Skip to content

Commit

Permalink
SDK-155: Add notebook API to Spark Command
Browse files Browse the repository at this point in the history
  • Loading branch information
@rahul26goyal @shahharsh87
rahul26goyal authored and shahharsh87 committed Mar 7, 2017
1 parent 4be1b43 commit 1fa62d2
Show file tree
Hide file tree
Showing 2 changed files with 62 additions and 8 deletions.
18 changes: 10 additions & 8 deletions qds_sdk/commands.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,8 @@ class SparkCommand(Command):

optparser.add_option("--sql", dest="sql", help="sql for Spark")

optparser.add_option("--note-id", dest="note_id", help="Id of the Notebook to run.")

optparser.add_option("-f", "--script_location", dest="script_location",
help="Path where spark program to run is stored. Has to be a local file path")

Expand Down Expand Up @@ -468,25 +470,25 @@ class SparkCommand(Command):
@classmethod
def validate_program(cls, options):
bool_program = options.program is not None
bool_other_options = options.script_location is not None or options.cmdline is not None or options.sql is not None
bool_other_options = options.script_location is not None or options.cmdline is not None or options.sql is not None or options.note_id is not None

# if both are false then no option is specified ==> raise ParseError
# if both are true then atleast two option specified ==> raise ParseError
if bool_program == bool_other_options:
raise ParseError("Exactly One of script location or program or cmdline or sql should be specified", cls.optparser.format_help())
raise ParseError("Exactly One of script location or program or cmdline or sql or note_id should be specified", cls.optparser.format_help())
if bool_program:
if options.language is None:
raise ParseError("Unspecified language for Program", cls.optparser.format_help())

@classmethod
def validate_cmdline(cls, options):
bool_cmdline = options.cmdline is not None
bool_other_options = options.script_location is not None or options.program is not None or options.sql is not None
bool_other_options = options.script_location is not None or options.program is not None or options.sql is not None or options.note_id is not None

# if both are false then no option is specified ==> raise ParseError
# if both are true then atleast two option specified ==> raise ParseError
if bool_cmdline == bool_other_options:
raise ParseError("Exactly One of script location or program or cmdline or sql should be specified", cls.optparser.format_help())
raise ParseError("Exactly One of script location or program or cmdline or sql or note_id should be specified", cls.optparser.format_help())
if bool_cmdline:
if options.language is not None:
raise ParseError("Language cannot be specified with the commandline option", cls.optparser.format_help())
Expand All @@ -496,25 +498,25 @@ def validate_cmdline(cls, options):
@classmethod
def validate_sql(cls, options):
bool_sql = options.sql is not None
bool_other_options = options.script_location is not None or options.program is not None or options.cmdline is not None
bool_other_options = options.script_location is not None or options.program is not None or options.cmdline is not None or options.note_id is not None

# if both are false then no option is specified => raise PraseError
# if both are true then atleast two option specified => raise ParseError
if bool_sql == bool_other_options:
raise ParseError("Exactly One of script location or program or cmdline or sql should be specified", cls.optparser.format_help())
raise ParseError("Exactly One of script location or program or cmdline or sql or note_id should be specified", cls.optparser.format_help())
if bool_sql:
if options.language is not None:
raise ParseError("Language cannot be specified with the 'sql' option", cls.optparser.format_help())

@classmethod
def validate_script_location(cls, options):
bool_script_location = options.script_location is not None
bool_other_options = options.program is not None or options.cmdline is not None or options.sql is not None
bool_other_options = options.program is not None or options.cmdline is not None or options.sql is not None or options.note_id is not None

# if both are false then no option is specified ==> raise ParseError
# if both are true then atleast two option specified ==> raise ParseError
if bool_script_location == bool_other_options:
raise ParseError("Exactly One of script location or program or cmdline or sql should be specified", cls.optparser.format_help())
raise ParseError("Exactly One of script location or program or cmdline or sql or note_id should be specified", cls.optparser.format_help())

if bool_script_location:
if options.language is not None:
Expand Down
52 changes: 52 additions & 0 deletions tests/test_command.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -390,8 +390,43 @@ def test_submit_query(self):
'user_program_arguments': None,
'can_notify': False,
'script_location': None,
'note_id' : None,
'retry': 0})

def test_submit_notebook(self):
sys.argv = ['qds.py', 'sparkcmd', 'submit', '--note-id', '111','--name', 'notebook-cmd']
print_command()
Connection._api_call = Mock(return_value={'id': 1234})
qds.main()
Connection._api_call.assert_called_with('POST', 'commands',
{'macros': None,
'label': None,
'language': None,
'tags': None,
'name': 'notebook-cmd',
'sql': None,
'program': None,
'app_id': None,
'cmdline':None,
'command_type': 'SparkCommand',
'arguments': None,
'user_program_arguments': None,
'can_notify': False,
'script_location': None,
'note_id' : '111',
'retry': 0})

def test_submit_notebook_with_cmdline(self):
sys.argv = ['qds.py', 'sparkcmd', 'submit', '--note-id', '111', '--cmdline', 'ls']
print_command()
with self.assertRaises(qds_sdk.exception.ParseError):
qds.main()
def test_submit_notebook_with_program(self):
sys.argv = ['qds.py', 'sparkcmd', 'submit', '--note-id', '111', '--program', 'print "hello"']
print_command()
with self.assertRaises(qds_sdk.exception.ParseError):
qds.main()

def test_submit_script_location_aws(self):
sys.argv = ['qds.py', 'sparkcmd', 'submit', '--script_location', 's3://bucket/path-to-script']
print_command()
Expand Down Expand Up @@ -421,6 +456,7 @@ def test_submit_script_location_local_py(self):
'user_program_arguments': None,
'can_notify': False,
'script_location': None,
'note_id' : None,
'retry': 0})

def test_submit_script_location_local_scala(self):
Expand All @@ -446,6 +482,7 @@ def test_submit_script_location_local_scala(self):
'user_program_arguments': None,
'can_notify': False,
'script_location': None,
'note_id' : None,
'retry': 0})

def test_submit_script_location_local_java(self):
Expand Down Expand Up @@ -480,6 +517,7 @@ def test_submit_script_location_local_R(self):
'user_program_arguments': None,
'can_notify': False,
'script_location': None,
'note_id' : None,
'retry': 0})

def test_submit_script_location_local_sql(self):
Expand All @@ -505,6 +543,7 @@ def test_submit_script_location_local_sql(self):
'user_program_arguments': None,
'can_notify': False,
'script_location': None,
'note_id' : None,
'retry': 0})

def test_submit_sql(self):
Expand All @@ -527,6 +566,7 @@ def test_submit_sql(self):
'user_program_arguments': None,
'can_notify': False,
'script_location': None,
'note_id' : None,
'retry': 0})

def test_submit_sql_with_language(self):
Expand Down Expand Up @@ -589,6 +629,7 @@ def test_submit_macros(self):
'cmdline': None,
'can_notify': False,
'script_location': None,
'note_id' : None,
'retry': 0})

def test_submit_tags(self):
Expand All @@ -612,6 +653,7 @@ def test_submit_tags(self):
'cmdline': None,
'can_notify': False,
'script_location': None,
'note_id' : None,
'retry': 0})

def test_submit_cluster_label(self):
Expand All @@ -635,6 +677,7 @@ def test_submit_cluster_label(self):
'command_type': 'SparkCommand',
'can_notify': False,
'script_location': None,
'note_id' : None,
'retry': 0})

def test_submit_name(self):
Expand All @@ -658,6 +701,7 @@ def test_submit_name(self):
'command_type': 'SparkCommand',
'can_notify': False,
'script_location': None,
'note_id' : None,
'retry': 0})

def test_submit_notify(self):
Expand All @@ -681,6 +725,7 @@ def test_submit_notify(self):
'user_program_arguments': None,
'can_notify': True,
'script_location': None,
'note_id' : None,
'retry': 0})

def test_submit_python_program(self):
Expand All @@ -703,6 +748,7 @@ def test_submit_python_program(self):
'user_program_arguments': None,
'can_notify': False,
'script_location': None,
'note_id' : None,
'retry': 0})

def test_submit_user_program_arguments(self):
Expand All @@ -728,6 +774,7 @@ def test_submit_user_program_arguments(self):
'user_program_arguments': 'world',
'can_notify': False,
'script_location': None,
'note_id' : None,
'retry': 0})

def test_submit_scala_program(self):
Expand All @@ -750,6 +797,7 @@ def test_submit_scala_program(self):
'user_program_arguments': None,
'can_notify': False,
'script_location': None,
'note_id' : None,
'retry': 0})

def test_submit_R_program(self):
Expand All @@ -772,6 +820,7 @@ def test_submit_R_program(self):
'user_program_arguments': None,
'can_notify': False,
'script_location': None,
'note_id' : None,
'retry': 0})

def test_submit_program_to_app(self):
Expand All @@ -795,6 +844,7 @@ def test_submit_program_to_app(self):
'user_program_arguments': None,
'can_notify': False,
'script_location': None,
'note_id' : None,
'retry': 0})

def test_submit_sql_to_app(self):
Expand All @@ -818,6 +868,7 @@ def test_submit_sql_to_app(self):
'user_program_arguments': None,
'can_notify': False,
'script_location': None,
'note_id' : None,
'retry': 0})

def test_submit_script_location_local_py_to_app(self):
Expand All @@ -844,6 +895,7 @@ def test_submit_script_location_local_py_to_app(self):
'user_program_arguments': None,
'can_notify': False,
'script_location': None,
'note_id' : None,
'retry': 0})

def test_submit_cmdline_to_app(self):
Expand Down

0 comments on commit 1fa62d2

Please sign in to comment.