Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Attributes from qstat command now found under native hash #198

Merged
merged 6 commits into from
Jul 9, 2020
Merged

Attributes from qstat command now found under native hash #198

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1271b7d
Attributes from qstat command now found under native hash
Jun 9, 2020
9dfe06d
Saves params under current job and saves all values under current_tex…
Jun 10, 2020
ee9a676
Added new native attributes for SGE
matthu017 Jun 30, 2020
be4434e
Adding native attributes to rspec tests
matthu017 Jul 1, 2020
3386e20
Changed variable name
matthu017 Jul 1, 2020
59e0eba
Add testcase for ST_name
matthu017 Jul 8, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 26 additions & 2 deletions lib/ood_core/job/adapters/sge/qstat_xml_j_r_listener.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,13 @@ def initialize
:tasks => [],
:status => :queued,
:procs => 1,
:native => {} # TODO: improve native attribute reporting
:native => {
:ST_name => ''
}
}
@current_text = nil
@current_request = nil
@processing_JB_stdout_path_list = false

@processing_job_array_spec = false
@adding_slots = false
Expand All @@ -42,6 +45,7 @@ def initialize
step: 1, # Step can have a default of 1
}
@running_tasks = []
@native_tags = ['JB_job_number', 'JB_job_name', 'JB_version', 'JB_project', 'JB_exec_file', 'JB_script_file', 'JB_script_size', 'JB_submission_time', 'JB_execution_time', 'JB_deadline', 'JB_owner', 'JB_uid', 'JB_group', 'JB_gid', 'JB_account', 'JB_cwd', 'JB_notify', 'JB_type', 'JB_reserve', 'JB_priority', 'JB_jobshare', 'JB_verify', 'JB_checkpoint_attr', 'JB_checkpoint_interval', 'JB_restart']
end

def tag_start(name, attrs)
Expand All @@ -50,10 +54,17 @@ def tag_start(name, attrs)
toggle_processing_array_spec
when 'JB_pe_range'
toggle_adding_slots
when 'JB_stdout_path_list'
@processing_JB_stdout_path_list = true
end
end

def tag_end(name)
#Add to native hash if in native_tags
if (@native_tags.include?(name))
@parsed_job[:native][:"#{name}"] = @current_text
end

case name
when 'JB_ja_tasks'
end_JB_ja_tasks
Expand Down Expand Up @@ -92,6 +103,10 @@ def tag_end(name)
toggle_processing_array_spec
when 'JB_pe_range'
toggle_adding_slots
when 'PN_path'
end_PN_path
when 'ST_name'
end_ST_name
end
end

Expand Down Expand Up @@ -153,6 +168,15 @@ def end_JAT_task_number
@running_tasks << @current_text
end

def end_PN_path
@parsed_job[:native][:PN_path] = @current_text if @processing_JB_stdout_path_list
@processing_JB_stdout_path_list = false
end

def end_ST_name
@parsed_job[:native][:ST_name] = @parsed_job[:native][:ST_name] + @current_text + ' '
end
johrstrom marked this conversation as resolved.
Show resolved Hide resolved

def set_job_array_piece(key)
@job_array_spec[key] = @current_text if @processing_job_array_spec
end
Expand Down Expand Up @@ -209,4 +233,4 @@ def set_slots
def ms_to_seconds(raw)
raw.digits.length >= 13 ? raw / 1000 : raw
end
end
end
27 changes: 25 additions & 2 deletions lib/ood_core/job/adapters/sge/qstat_xml_r_listener.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,21 +24,32 @@ def initialize
@parsed_jobs = []
@current_job = {
:tasks => [],
:native => {} # TODO: improve native reporting
:native => {
:ST_name => ''
}
}
@current_text = nil
@processing_JB_stdout_path_list = false

@current_request = nil
@native_tags = ['JB_job_number', 'JB_job_name', 'JB_version', 'JB_project', 'JB_exec_file', 'JB_script_file', 'JB_script_size', 'JB_submission_time', 'JB_execution_time', 'JB_deadline', 'JB_owner', 'JB_uid', 'JB_group', 'JB_gid', 'JB_account', 'JB_cwd', 'JB_notify', 'JB_type', 'JB_reserve', 'JB_priority', 'JB_jobshare', 'JB_verify', 'JB_checkpoint_attr', 'JB_checkpoint_interval', 'JB_restart']
johrstrom marked this conversation as resolved.
Show resolved Hide resolved
end

def tag_start(name, attributes)
case name
when 'hard_request'
start_hard_request(attributes)
when "JB_stdout_path_list"
@processing_JB_stdout_path_list = true
end
end

def tag_end(name)
#Add text if in native_tags
if (@native_tags.include?(name))
@current_job[:native][:"#{name}"] = @current_text
end

case name
when 'job_list'
end_job_list
Expand All @@ -64,6 +75,10 @@ def tag_end(name)
end_hard_request
when 'tasks'
add_child_tasks
when 'PN_path'
end_PN_path
when 'ST_name'
end_ST_name
end
end

Expand Down Expand Up @@ -130,6 +145,15 @@ def end_hard_request
end
end

def end_PN_path
@current_job[:native][:PN_path] = @current_text if @processing_JB_stdout_path_list
@processing_JB_stdout_path_list = false
end

def end_ST_name
@current_job[:native][:ST_name] = @current_job[:native][:ST_name] + @current_text + ' '
end

# Store a completed job and reset current_job for the next pass
def end_job_list
@parsed_jobs << @current_job
Expand All @@ -145,4 +169,3 @@ def add_child_tasks
}
end
end

118 changes: 111 additions & 7 deletions spec/job/adapters/sge/batch_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,12 @@ def load_resource_file(file_name)
:dispatch_time => DateTime.parse('2018-10-10T14:37:16').to_time.to_i,
:wallclock_limit => 360,
:wallclock_time => Time.now.to_i - DateTime.parse('2018-10-10T14:37:16').to_time.to_i,
:native => {}
:native => {
:ST_name=>"",
:JB_job_number=>"88",
:JB_owner=>"vagrant",
:JB_project=>"project_a"
}
),
OodCore::Job::Info.new( # Queued job, w/ project
:id => '1045',
Expand All @@ -33,7 +38,12 @@ def load_resource_file(file_name)
:submission_time => DateTime.parse('2018-10-09T18:47:05').to_time.to_i,
:wallclock_limit => 360,
:wallclock_time => 0,
:native => {}
:native => {
:JB_job_number=>"1045",
:JB_owner=>"vagrant",
:JB_project=>"project_b",
:JB_submission_time=>"2018-10-09T18:47:05"
}
),
OodCore::Job::Info.new( # Queued job w/o project
:id => '1046',
Expand All @@ -45,7 +55,11 @@ def load_resource_file(file_name)
:submission_time => DateTime.parse('2018-10-09T18:47:05').to_time.to_i,
:wallclock_limit => 360,
:wallclock_time => 0,
:native => {},
:native => {
:JB_job_number=>"1046",
:JB_owner=>"vagrant",
:JB_submission_time=>"2018-10-09T18:47:05"
},
:tasks => [
{ :id => '1', :status => :queued },
{ :id => '3', :status => :queued },
Expand All @@ -64,7 +78,11 @@ def load_resource_file(file_name)
:submission_time => DateTime.parse('2018-10-09T18:35:12').to_time.to_i,
:wallclock_limit => 360,
:wallclock_time => 0,
:native => {}
:native => {
:JB_job_number=>"44",
:JB_owner=>"vagrant",
:JB_submission_time=>"2018-10-09T18:35:12"
}
)
]}

Expand All @@ -77,7 +95,34 @@ def load_resource_file(file_name)
:id => "4147342",
:job_name => "cfSNV_0merged_split_pileup_241.FLASH.recal_10.pbs",
:job_owner => "shuoli",
:native => {},
:native =>{
:ST_name=>"",
:JB_job_number=>"4147342",
:JB_exec_file=>"job_scripts/4147342",
:JB_submission_time=>"1541444183",
:JB_owner=>"shuoli",
:JB_uid=>"13287",
:JB_group=>"xjzhou",
:JB_gid=>"12426",
:JB_account=>"sge",
:JB_project=>"xjzhou_prj",
:JB_notify=>"false",
:JB_job_name=>"cfSNV_0merged_split_pileup_241.FLASH.recal_10.pbs",
:JB_jobshare=>"0",
:JB_script_file=>"cfSNV_0merged_split_pileup_241.FLASH.recal_10.pbs",
:JB_cwd=>"/u/project/xjzhou/shuoli/Zo_WES_plasma_SC1810033_10312018/code/0",
:JB_deadline=>"0",
:JB_execution_time=>"0",
:JB_checkpoint_attr=>"0",
:JB_checkpoint_interval=>"0",
:JB_reserve=>"false",
:JB_priority=>"1024",
:JB_restart=>"0",
:JB_verify=>"0",
:JB_script_size=>"0",
:JB_version=>"0",
:JB_type=>"0"
},
:procs => 1,
:queue_name => nil,
:status => :running,
Expand All @@ -97,7 +142,34 @@ def load_resource_file(file_name)
:id => "942195",
:job_name => "RemoteDesktop",
:job_owner => "smatott",
:native => {},
:native => {
:ST_name=>"",
:JB_job_number=>"942195",
:JB_exec_file=>"job_scripts/942195",
:JB_submission_time=>"1576525304",
:JB_owner=>"smatott",
:JB_uid=>"1105",
:JB_group=>"packages",
:JB_gid=>"3002",
:JB_account=>"sge",
:JB_notify=>"false",
:JB_job_name=>"RemoteDesktop",
:PN_path=> "/mnt/lustre/users/smatott/ondemand/data/sys/dashboard/batch_connect/sys/bc_desktop/hpc/output/9beca517-07fa-4379-8eb7-934460846a19/output.log",
:JB_jobshare=>"0",
:JB_script_file=>"STDIN",
:JB_cwd=> "/mnt/lustre/users/smatott/ondemand/data/sys/dashboard/batch_connect/sys/bc_desktop/hpc/output/9beca517-07fa-4379-8eb7-934460846a19",
:JB_deadline=>"0",
:JB_execution_time=>"0",
:JB_checkpoint_attr=>"0",
:JB_checkpoint_interval=>"0",
:JB_reserve=>"false",
:JB_priority=>"1024",
:JB_restart=>"0",
:JB_verify=>"0",
:JB_script_size=>"0",
:JB_version=>"0",
:JB_type=>"0"
},
:procs => 16,
:queue_name => 'all.q',
:status => :running,
Expand All @@ -117,7 +189,35 @@ def load_resource_file(file_name)
:id => "748172",
:job_name => "jupyter_interactive",
:job_owner => "johrstrom",
:native => {},
:native => {
:ST_name=>"/export/uge/bin/lx-amd64/qsub -wd /home/johrstrom/ondemand/data/sys/dashboard/batch_connect/sys/jupyter/output/a4a46499-77d9-4334-bef2-71dd0a0857f6 -N jupyter_interactive -o /home/johrstrom/ondemand/data/sys/dashboard/batch_connect/sys/jupyter/output/a4a46499-77d9-4334-bef2-71dd0a0857f6/output.log -q ondemand -l h_rt=04:00:00 -P communitycluster -V -pe sm 11 hpcc ",
:JB_job_number=>"748172",
:JB_job_name=>"jupyter_interactive",
:JB_version=>"0",
:JB_project=>"communitycluster",
:JB_exec_file=>"job_scripts/748172",
:JB_script_file=>"STDIN",
:JB_script_size=>"0",
:JB_submission_time=>"1592928409331",
:JB_execution_time=>"0",
:JB_deadline=>"0",
:JB_owner=>"johrstrom",
:JB_uid=>"99577",
:JB_group=>"hpcc",
:JB_gid=>"101",
:JB_account=>"sge",
:JB_cwd=>"/home/johrstrom/ondemand/data/sys/dashboard/batch_connect/sys/jupyter/output/a4a46499-77d9-4334-bef2-71dd0a0857f6",
:JB_notify=>"false",
:JB_type=>"0",
:JB_reserve=>"false",
:JB_priority=>"0",
:JB_jobshare=>"0",
:JB_verify=>"0",
:JB_checkpoint_attr=>"0",
:JB_checkpoint_interval=>"0",
:JB_restart=>"0",
:PN_path=>"/home/johrstrom/ondemand/data/sys/dashboard/batch_connect/sys/jupyter/output/a4a46499-77d9-4334-bef2-71dd0a0857f6/output.log"
},
:procs => 11,
:queue_name => 'ondemand',
:status => :running,
Expand Down Expand Up @@ -210,6 +310,10 @@ def load_resource_file(file_name)
it "expects to receive the correct job info" do
expect(batch.get_info_enqueued_job('748172') ).to eq(job_from_uge_qstat_jr)
end

it "expects to receive the correct ST_name" do
expect(batch.get_info_enqueued_job('748172').native[:ST_name] ).to eq("/export/uge/bin/lx-amd64/qsub -wd /home/johrstrom/ondemand/data/sys/dashboard/batch_connect/sys/jupyter/output/a4a46499-77d9-4334-bef2-71dd0a0857f6 -N jupyter_interactive -o /home/johrstrom/ondemand/data/sys/dashboard/batch_connect/sys/jupyter/output/a4a46499-77d9-4334-bef2-71dd0a0857f6/output.log -q ondemand -l h_rt=04:00:00 -P communitycluster -V -pe sm 11 hpcc ")
end
end
end

Expand Down