-
Notifications
You must be signed in to change notification settings - Fork 0
/
batch_submit_manyfilesperjob.py
executable file
·79 lines (75 loc) · 2.92 KB
/
batch_submit_manyfilesperjob.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
#! /usr/bin/env python
import os
import sys
if len(sys.argv) != 5:
print "usage batch_submit_manyfilesperjob.py dataset njobs applicationName queue"
sys.exit(1)
dataset = sys.argv[1]
inputlist = "castor_09prod/"+dataset+".list"
output = dataset
# queue
queue = sys.argv[4]
#ijobmax = 40
ijobmax = int(sys.argv[2])
#application = "JPsiApp"
application = sys.argv[3]
################################################
castordir = "/castor/cern.ch/user/nonserve/"
diskoutputdir = "/cmsrm/pc21/crovelli/data/JPsi3.X/"
outputmain = castordir+output
diskoutputmain = diskoutputdir+output
################################################
os.system("mkdir "+output)
os.system("mkdir "+output+"/log/")
os.system("mkdir "+output+"/input/")
os.system("mkdir "+output+"/src/")
outputroot = outputmain+"/root/"
if castordir != "none":
os.system("rfmkdir "+outputmain)
os.system("rfmkdir "+outputroot)
os.system("rfchmod 777 "+outputmain)
os.system("rfchmod 777 "+outputroot)
else: os.system("mkdir "+outputroot)
if diskoutputdir != "none":
os.system("ssh -o BatchMode=yes -o StrictHostKeyChecking=no pccmsrm21 mkdir "+diskoutputmain)
#look for the current directory
#######################################
pwd = os.environ['PWD']
#######################################
numfiles = reduce(lambda x,y: x+1, file(inputlist).xreadlines(), 0)
filesperjob = numfiles/ijobmax
extrafiles = numfiles%ijobmax
input = open(inputlist)
######################################
for ijob in range(ijobmax):
# prepare the list file
inputfilename = pwd+"/"+output+"/input/input_"+str(ijob)+".list"
inputfile = open(inputfilename,'w')
# if it is a normal job get filesperjob lines
if ijob != (ijobmax-1):
for line in range(filesperjob):
ntpfile = input.readline()
inputfile.write(ntpfile)
continue
else:
# if it is the last job get ALL remaining lines
ntpfile = input.readline()
while ntpfile != '':
inputfile.write(ntpfile)
ntpfile = input.readline()
continue
inputfile.close()
# prepare the script to run
outputname = output+"/src/submit_"+str(ijob)+".src"
outputfile = open(outputname,'w')
outputfile.write('#!/bin/bash\n')
outputfile.write('cp -r '+pwd+'/config $WORKDIR\n')
outputfile.write('cd $WORKDIR\n')
outputfile.write(pwd+'/'+application+' '+inputfilename+" "+output+"_"+str(ijob)+"_ \n")
outputfile.write('ls *.root | grep -v Z_calibFall08 | xargs -i rfcp {} '+outputroot+'\n')
outputfile.write('ls *.root | grep -v Z_calibFall08 | xargs -i scp -o BatchMode=yes -o StrictHostKeyChecking=no {} pccmsrm21:'+diskoutputmain+'/{}\n')
outputfile.close
os.system("echo bsub -q "+queue+" -o "+output+"/log/"+output+"_"+str(ijob)+".log source "+pwd+"/"+outputname)
os.system("bsub -q "+queue+" -o "+output+"/log/"+output+"_"+str(ijob)+".log source "+pwd+"/"+outputname)
ijob = ijob+1
continue