Improve judging errors on the submission page

.github/workflows/ci.yml

@@ -46,7 +46,7 @@ jobs:
           SCHEDULE_BACKUPS: 0
           ISOLATE_ROOT: /
           ISOLATE_CGROUPS: false
-          ISOLATE_BRANCH: master
+          ISOLATE_BRANCH: v1.10.1
 
       - name: Back up db/schema.rb # it will be overwritten when install.bash runs migrate.bash; back up the original so we can check if it's up to date
         run: cp db/schema.rb db/schema.rb.git

app/assets/stylesheets/submission.css

@@ -19,6 +19,7 @@ table.results {
   border-spacing: 0;
   border-collapse: collapse;
   width: 100%;
+  margin: 15px 0px;
 }
 
 .results th {

app/controllers/evaluators_controller.rb

@@ -2,7 +2,7 @@ class EvaluatorsController < ApplicationController
 
   def permitted_params
     @_permitted_params ||= begin
-      permitted_attributes = [:name, :description, :source]
+      permitted_attributes = [:name, :description, :source, :language_id]
       permitted_attributes << :owner_id if policy(@evaluator || Evaluator).transfer?
       params.require(:evaluator).permit(*permitted_attributes)
     end

app/models/evaluator.rb

@@ -3,6 +3,7 @@ class Evaluator < ActiveRecord::Base
 
   has_many :problems
   belongs_to :owner, :class_name => :User
+  belongs_to :language
 
   validates :name, :presence => true
 

app/models/submission.rb

@@ -149,7 +149,7 @@ def update_test_messages
 
       judge_data = self.judge_data
       return if judge_data.status == :pending # incomplete judge_log
-      return if judge_data.errored? # judge errored - very bad
+      return if judge_data.data.has_key?('error') # judge errored - very bad
 
       errors = []
       warnings = []

app/models/submission/judge_data.rb

@@ -274,7 +274,7 @@ def initialize(log, test_sets, test_cases, prerequisite_sets = [])
     end
 
     def errored?
-      data.has_key?('error')
+      data.has_key?('error') || status == :error
     end
 
     def completed?
@@ -285,10 +285,18 @@ def compiled?
       data.has_key?('compile')
     end
 
+    def evaluator_compiled?
+      data.has_key?('evaluator_compile')
+    end
+
     def compilation
       @compilation ||= Compilation.new(data['compile'])
     end
 
+    def evaluator_compilation
+      @evaluator_compilation ||= Compilation.new(data['evaluator_compile'])
+    end
+
     def prerequisite_sets
       test_sets.slice(@presets)
     end

app/views/evaluators/_form.html.erb

@@ -23,6 +23,10 @@
     <%= f.label :source %><br />
     <%= f.text_area :source %>
   </div>
+  <div class="field">
+    <%= f.label :language_id %><br>
+    <%= f.select :language_id, grouped_options_for_select(Language.grouped_submission_options, @evaluator.language_id), :include_blank => true %>
+  </div>
   <div class="field">
     <%= f.label :owner_id %><br />
     <% if policy(@evaluator).transfer? %>

app/views/evaluators/index.html.erb

@@ -7,6 +7,7 @@
       <th>Name</th>
       <th>Description</th>
       <th>User</th>
+      <th>Langauge</th>
       <th></th>
       <% if policy(Evaluator).update? %>
         <th></th>
@@ -22,6 +23,7 @@
     <td><%= evaluator.name %></td>
     <td><%= evaluator.description %></td>
     <td><%= evaluator.owner_id %></td>
+    <td><%= evaluator.language&.name %></td>
     <td><%= link_to 'Show', evaluator %></td>
     <% if policy(Evaluator).update? %>
       <td><%= link_to 'Edit', edit_evaluator_path(evaluator) if policy(evaluator).update? %></td>

app/views/evaluators/show.html.erb

@@ -8,8 +8,10 @@
 </p>
 <% if policy(@evaluator).inspect? %>
   <p>
+    <b>Language:</b>
+    <%= @evaluator.language&.name %><br>
     <b>Source:</b>
-    <pre><%= @evaluator.source %></pre>
+    <%= predisplay @evaluator.source, language: @evaluator.language&.lexer %>
   </p>
 <% end %>
 

app/views/problems/_admin.html.erb

@@ -54,7 +54,7 @@
   <% if @problem.evaluator %>
     <%= link_to @problem.evaluator.name, @problem.evaluator %>
     <% if policy(@problem.evaluator).inspect? # privilege required to see evaluator source %>
-      <%= predisplay @problem.evaluator.source, language: :sh %>
+      <%= predisplay @problem.evaluator.source, language: @problem.evaluator.language&.lexer %>
     <% end %>
   <% else %>
     Default evaluator

app/views/submissions/show.html.erb

@@ -5,7 +5,12 @@
 
 <%= stylesheet_link_tag "submission" %>
 
-<% if @submission.score == nil %>
+<% @judge_data = @submission.judge_data %>
+<% if @judge_data.errored? %>
+  <p>
+    <b>An unexpected error has occurred during judging. Please retry the submission, or contact us at <%= mail_to "[email protected]" %> if the error persists.</b>
+  </p>
+<% elsif @submission.score == nil %>
   <p>
     <b>This submission has not finished judging. Refresh this page in a minute or two to see the submission's score.</b>
   </p>
@@ -46,9 +51,8 @@
   <% end %>
 </p>
 
-<% @judge_data = @submission.judge_data %>
-<table class="results">
-  <% if @judge_data.compiled? %>
+<% if @judge_data.compiled? %>
+  <table class="results">
     <tbody class="compilation status_<%= @judge_data.compilation.status %>">
       <tr>
         <th class="compile_command" colspan="3">Compilation: <span class="code"><%= @judge_data.compilation.command %></span></th>
@@ -59,11 +63,24 @@
         <td colspan="5"><span class="code"><%= @judge_data.compilation.log %></span></td>
       </tr>
     </tbody>
-  <% end %>
-</table>
-&nbsp;
+  </table>
+<% end %>
+<% if @judge_data.evaluator_compiled? && @submission.problem.evaluator && policy(@submission.problem.evaluator).inspect? %>
+  <table class="results">
+    <tbody class="compilation status_<%= @judge_data.evaluator_compilation.status %>">
+      <tr>
+        <th class="compile_command" colspan="3">Evaluator Compilation: <span class="code"><%= @judge_data.evaluator_compilation.command %></span></th>
+        <th class="compile_status"><%= @judge_data.evaluator_compilation.judgement %></th>
+        <th>&nbsp;</th>
+      </tr>
+      <tr>
+        <td colspan="5"><span class="code"><%= @judge_data.evaluator_compilation.log %></span></td>
+      </tr>
+    </tbody>
+  </table>
+<% end %>
 <table class="results">
-  <% if !@judge_data.compiled? || @judge_data.compilation.status == :success %>
+  <% if !@judge_data.errored? && (!@judge_data.compiled? || @judge_data.compilation.status == :success) %>
     <tbody class="headings">
       <th class="test_name"></th>
       <th>Time</th>
@@ -152,8 +169,6 @@
     </tr>
   </tbody>
 </table>
-<br>
-<br>
 <p>
 <b>Source:</b>
 <%= predisplay(@submission.source || "", language: @submission.language.lexer) %>

app/workers/judge_submission_worker.rb

@@ -46,13 +46,13 @@ def perform(submission_id)
   rescue StandardError => e
     unless self.submission.nil?
       submission.reload
-      submission.judge_log = {'error' => {'message' => e.message, 'backtrace' => e.backtrace}}.to_json
+      submission.judge_log = {'error' => {'message' => e.message, 'backtrace' => e.backtrace}, 'status' => 2}.to_json
       submission.save
     end
     raise
   end
 
-  EvalFileName = "eval.sh"
+  EvalFileName = "eval"
   OutputBaseLimit = 1024 * 1024 * 2
 
   attr_accessor :submission, :exe_filename
@@ -68,14 +68,34 @@ def judge
     result = {}
     setup_judging do
       if submission.language.compiled
-        result['compile'] = compile!(exe_filename) # possible caching
-        return result.merge!(grade_compile_error(result['compile'])) if result['compile']['stat'] != 0 #error
+        result['compile'] = compile!(submission.source, submission.language, exe_filename) # possible caching
+        return result.merge!(grade_compile_error(result['compile'])) if result['compile']['stat'] != 0 # error
       else
         File.open(File.expand_path(exe_filename, tmpdir),"w") { |f| f.write(submission.source) }
       end
 
       run_command = submission.language.run_command(exe_filename)
 
+      if problem.evaluator.nil?
+        eval_command = nil
+      else
+        if problem.evaluator.language.nil?
+          eval_command = "./#{EvalFileName}"
+        else
+          eval_command = problem.evaluator.language.run_command(EvalFileName)
+        end
+
+        if problem.evaluator.language&.compiled
+          evaluator_compilation = compile!(problem.evaluator.source, problem.evaluator.language, EvalFileName) # possible caching
+          return result.merge!('evaluator_compile' => evaluator_compilation, 'status' => 2) if evaluator_compilation['stat'] != 0 # error
+        else
+          File.open(File.expand_path(EvalFileName, tmpdir),"w") do |file|
+            file.chmod(0700)
+            file.write(problem.evaluator.source.gsub(/\r\n?/, "\n"))
+          end
+        end
+      end
+
       result['test_cases'] = {}
       result['test_sets'] = {}
 
@@ -85,7 +105,7 @@ def judge
       prereqs = problem.test_cases.where(:id => problem.prerequisite_sets.joins(:test_case_relations).select(:test_case_relations => :test_case_id))
 
       prereqs.each do |test_case|
-        result['test_cases'][test_case.id] = judge_test_case(test_case, run_command, resource_limits) unless result['test_cases'].has_key?(test_case.id)
+        result['test_cases'][test_case.id] = judge_test_case(test_case, run_command, eval_command, resource_limits) unless result['test_cases'].has_key?(test_case.id)
       end
 
       problem.prerequisite_sets.each do |test_set|
@@ -100,7 +120,7 @@ def judge
 
       # test cases
       (problem.test_cases - prereqs).each do |test_case|
-        result['test_cases'][test_case.id] = judge_test_case(test_case, run_command, resource_limits) unless result['test_cases'].has_key?(test_case.id)
+        result['test_cases'][test_case.id] = judge_test_case(test_case, run_command, eval_command, resource_limits) unless result['test_cases'].has_key?(test_case.id)
       end
 
       # test sets
@@ -148,18 +168,18 @@ def setup_judging
     end
   end
 
-  def compile! output
-    result = submission.language.compile(box, submission.source, output, :mem => 393216, :wall_time => 60)
+  def compile!(source, language, output)
+    result = language.compile(box, source, output, :mem => 393216, :wall_time => 60)
     FileUtils.copy(box.expand_path(output), File.expand_path(output, tmpdir)) if result['stat'] == 0
     return result
   ensure
     box.clean!
   end
 
-  def judge_test_case(test_case, run_command, resource_limits)
+  def judge_test_case(test_case, run_command, eval_command, resource_limits)
     FileUtils.copy(File.expand_path(exe_filename, tmpdir), box.expand_path(exe_filename))
     result = run_test_case(test_case, run_command, resource_limits)
-    result['evaluator'] = evaluate_output(test_case, result['output'], result['output_size'], problem.evaluator)
+    result['evaluator'] = evaluate_output(test_case, result['output'], result['output_size'], eval_command)
     result['log'] = truncate_output(result['log']) # log only a small portion
     result['output'] = truncate_output(result['output'].slice(0,100)) # store only a small portion
     result
@@ -190,21 +210,18 @@ def run_test_case(test_case, run_command, resource_limits = {})
     box.clean!
   end
 
-  def evaluate_output(test_case, output, output_size, evaluator)
+  def evaluate_output(test_case, output, output_size, eval_command)
     stream_limit = OutputBaseLimit + test_case.output.bytesize*2
     if output_size > stream_limit
       return {'evaluation' => 0, 'log' => "Output exceeded the streamsize limit of #{stream_limit}.", 'meta' => {'status' => 'OK'}}
     end
     expected = conditioned_output(test_case.output)
     actual = conditioned_output(output)
-    if evaluator.nil?
+    if eval_command.nil?
       {'evaluation' => (actual == expected ? 1 : 0), 'meta' => {'status' => 'OK'}}
     else
       r = {}
-      box.fopen(EvalFileName,"w") do |file|
-        file.chmod(0700)
-        file.write(problem.evaluator.source.gsub(/\r\n?/, "\n"))
-      end
+      FileUtils.copy(File.expand_path(EvalFileName, tmpdir), box.expand_path(EvalFileName))
       resource_limits = { :mem => 524288, :time => time_limit*3+15, :wall_time => time_limit*3+30 }
       box.fopen("actual","w") { |f| f.write(actual) } # DEPRECATED
       box.fopen("input","w") { |f| f.write(test_case.input) } # DEPRECATED
@@ -213,7 +230,7 @@ def evaluate_output(test_case, output, output_size, evaluator)
       eval_output = nil
       str_to_pipe(test_case.input, expected) do |input_stream, output_stream|
         run_opts = resource_limits.reverse_merge(:processes => true, 3 => input_stream, 4 => output_stream, :stdin_data => actual, :output_limit => OutputBaseLimit + test_case.output.bytesize*4, :clean_utf8 => true, :inherit_fds => true)
-        (stdout,), (r['log'],r['log_size']), (r['box'],), r['meta'], status = box.capture5("./#{EvalFileName} #{deprecated_args}", run_opts )
+        (stdout,), (r['log'],r['log_size']), (r['box'],), r['meta'], status = box.capture5("#{eval_command} #{deprecated_args}", run_opts )
         r['log'] = truncate_output(r['log'])
         return r.merge('stat' => 2, 'box' => 'Output was not a valid UTF-8 encoding\n'+r['box']) if !output.force_encoding("UTF-8").valid_encoding?
         eval_output = stdout.strip.split(nil,2)

db/migrate/20230225054132_add_language_id_to_evaluators.rb

@@ -0,0 +1,5 @@
+class AddLanguageIdToEvaluators < ActiveRecord::Migration
+  def change
+    add_column :evaluators, :language_id, :integer
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20200418113601) do
+ActiveRecord::Schema.define(version: 20230225054132) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -93,6 +93,7 @@
     t.integer  "owner_id",                 null: false
     t.datetime "created_at"
     t.datetime "updated_at"
+    t.integer  "language_id"
   end
 
   create_table "file_attachments", force: :cascade do |t|

script/install/config.bash

@@ -203,7 +203,7 @@ declare -p ISOLATE_CGROUPS &> /dev/null || while [ -z "$ISOLATE_CGROUPS" ] ; do
   else ISOLATE_CGROUPS=false; fi
 done
 
-declare -p ISOLATE_BRANCH &> /dev/null || ISOLATE_BRANCH=master # no prompt
+declare -p ISOLATE_BRANCH &> /dev/null || ISOLATE_BRANCH=v1.10.1 # no prompt
 
 
 shopt -u nocasematch;

script/install/isolate.bash

@@ -9,7 +9,7 @@ cd $srclocation
 if [ -d "isolate" ]; then
   cd isolate
   done=true
-  git pull --force | grep -q -v 'Already up-to-date.' && done=false
+  # git pull --force | grep -q -v 'Already up-to-date.' && done=false
   if $done; then
     exit
   fi