cmsRun within Condor
A few different scripts used to generate condor_submit and CMSSW configurations for
submitting CMSSW jobs to our condor cluster.
Example usage:
perl condor_filelist.perl \
python/ConfFile_MyCmsConfif_cfg.py \
datafiles/ListOfFiles.txt \
{CMS run options} \
--prodSpace /local/cms/user/{username}/jobName \
--batch 10 \
--jobname JobName
You must run cmsenv (in your desired CMSSW distribution!) before calling condor_filelist.perl.
condor_filelist.perl will then group the files in your input file list into batches based
on the batch size, and generate cfg files for each job in <prodSpace>/<jobName>/cfg.
Upon submission to condor, batch_cmsRun (which expects to be moved to ~/bin/batch_cmsRun)
will be called for each job to set up the environment and run cmssw.
This version is set up for running on the UMN cluster, and uses the built-in file transferring
utilities of condor to move the input files. In addition, each job generates a new executable
named after the job name which calls batch_cmsRun from within a slc7 cmssw singularity container.
batch_cmsRun
#!/bin/sh
SCRAM_ARCH=$1
RELSPACE=$2
WORKSPACE=$3
FILE=$4
STARTDIR=$(pwd)
echo ${STARTDIR}
if [ $# -gt "4" ]
then
LOG=$5
else
LOG=${FILE%%.cfg}.log
fi
if [ $# -gt "5" ]
then
ELOG=$6
else
ELOG=${FILE%%.cfg}.elog
fi
if [ $# -gt "6" ]
then
/bin/rm -f $7
fi
CMSRUN_ARGUMENTS=""
for var in "${@:9}"
do
CMSRUN_ARGUMENTS="${CMSRUN_ARGUMENTS} $var"
done
mkdir -p ${WORKSPACE}
cd ${WORKSPACE}
/bin/hostname
echo "INSIDE batch_cmsRun"
echo ${RELSPACE}
echo ${CMSRUN_ARGUMENTS}
pwd
# This version uses the locally installed versions of CMSSW
#source /local/cms/sw/cmsset_${SCRAM_ARCH}.sh >> ${LOG} 2>> ${ELOG}
# This version uses the remotely mounted drives at CERN
#echo "TESTING" >> ${LOG}
#set >> ${LOG}
cd ${RELSPACE}
#echo ======================== >> ${LOG}
#scram runtime -sh >> ${LOG} 2>>${ELOG}
#echo ======================== >> ${LOG}
eval 'cmsenv'
#echo ======================== >> ${LOG}
#set >> ${LOG}
cd ${STARTDIR}
cmsRun ${FILE} ${CMSRUN_ARGUMENTS}
condor_filelist.perl
#!/usr/bin/perl
use Getopt::Long;
use File::Basename;
#------------------------
#$prodSpace=$ENV{"HOME"}."/work";
$prodSpace="/data/whybee0b/user/".$ENV{"USER"};
$batch=10;
$startPoint=0;
$nosubmit='';
$use_xrootd=''; # '' is false in perl
$rt=$ENV{"LOCALRT"};
$arch=$ENV{"SCRAM_ARCH"};
$jobBase="default";
GetOptions(
"batch=i" => \$batch,
"start=i" => \$startPoint,
"nosubmit" => \$nosubmit,
"prodspace=s" => \$prodSpace,
"jobname=s" => \$jobBase,
"xrootd" => \$use_xrootd,
"nice" => \$nice_user,
"resubmit" => \$resub
);
$executable="$prodSpace/$jobBase/cfg/$jobBase";
print "$executable\n";
open(OUTP,">$executable");
print OUTP "#!/bin/sh\n";
print OUTP "source /local/cms/sw/cmsset_default.sh\n";
print OUTP "pwd\n";
print OUTP "hostname\n";
print OUTP "export SINGULARITY_BIND=\"/local/,/export/scratch/,/home/\"\n";
print OUTP "cmssw-cc7 --command-to-run ".$ENV{"HOME"}."/bin/batch_cmsRun \$@";
close(OUTP);
chmod 0755,$executable;
print "$#ARGV\n";
$nargs = $#ARGV;
$basecfg=shift @ARGV;
$filelist=shift @ARGV;
$cmsRunArguments=shift @ARGV;
for(my $i = 0; $i <= $nargs-3; $i++)
{
$nextArg=shift @ARGV;
$cmsRunArguments=$cmsRunArguments." ".$nextArg;
}
print "cmsRun Arguments: $cmsRunArguments\n";
if ($jobBase eq "default") {
my $stub3=$basecfg;
$stub3=~s|.*/||g;
$stub3=~s|_cfg.py||;
$stub3=~s|[.]py||;
$jobBase=$stub3;
}
if (length($rt)<2) {
print "You must run \"cmsenv\" in the right release area\n";
print "before running this script!\n";
exit(1);
}
if ($use_xrootd) {
# Try to find the user's proxy file
open(VOMSY,"voms-proxy-info|");
while (<VOMSY>) {
if (/path\s+:\s+(\S+)/) {
$voms_proxy=$1;
}
}
close(VOMSY);
}
#------------------------
print "Setting up a job based on $basecfg into $jobBase using $filelist\n";
if ($nosubmit) {
print " Will not actually submit this job\n";
}
if ($resub) {
print " Resubmitting failed jobs\n";
} else {
$cfg=$basecfg;
system("mkdir -p $prodSpace/$jobBase");
mkdir("$prodSpace/$jobBase/cfg");
mkdir("$prodSpace/$jobBase/log");
}
$linearn=0;
srand(); # make sure rand is ready to go
if ($nosubmit) {
open(SUBMIT,">condor_submit.txt");
} else {
open(SUBMIT,"|condor_submit");
}
print(SUBMIT "Executable = $executable\n");
print(SUBMIT "Arguments = \"$cmsRunArguments\"\n");
print(SUBMIT "Universe = vanilla\n");
print(SUBMIT "initialdir = $prodSpace/$jobBase\n");
print(SUBMIT "Output = $prodSpace/$jobBase/log/output.\$(Process)\n");
print(SUBMIT "Error = $prodSpace/$jobBase/log/error.\$(Process)\n");
print(SUBMIT "request_memory = 1G\n");
print(SUBMIT "should_transfer_files = YES\n");
if ($use_xrootd) {
# If the proxy file exists and is a normal file, we use it
if (-f $voms_proxy) {
print("Found voms proxy: $voms_proxy\n");
print(SUBMIT "should_transfer_files = YES\n");
print(SUBMIT "transfer_input_files = $voms_proxy\n");
print(SUBMIT "X509UserProxy = $voms_proxy\n");
}
# Invalid file
else {
print("No voms proxy found! Please run `voms-proxy-init` and confirm that the file exists at /tmp/x509*\n");
exit(1);
}
}
if ($nice_user) {
print(SUBMIT "nice_user = True\n");
}
open(FLIST,$filelist);
while (<FLIST>) {
chomp;
push @flist,$_;
}
close(FLIST);
$i=0;
$ii=$startPoint-1;
while ($i<=$#flist) {
$ii++;
@jobf=();
for ($j=0; $j<$batch && $i<=$#flist; $j++) {
if ($flist[$i]=~/file:/){
$flist[$i]=~s|file:||;
}
push @jobf,$flist[$i];
$i++;
}
if($resub)
{
$stub2=$jobBase;
$stub2.=sprintf("_%03d",$ii);
$fname=$stub2.".root";
$jobCfg="$prodSpace/$jobBase/cfg/".$stub2."_cfg.py";
unless(-e "$prodSpace/$jobBase/$fname") {print("Resubmitting $stub2\n")}
} else {
$jobCfg=specializeCfg($cfg,$ii,@jobf);
}
unless($resub and -e "$prodSpace/$jobBase/$fname")
{
$stub=$jobCfg;
$stub=~s|.*/([^/]+)_cfg.py$|$1|;
$log="$prodSpace/$jobBase/log/$stub.log";
$elog="$prodSpace/$jobBase/log/$stub.err";
$sleep=(($ii*2) % 60)+2; # Never sleep more than a ~minute, but always sleep at least 2
print(SUBMIT "Arguments = $arch $rt $prodSpace/$jobBase $jobCfg $log $elog $fname $sleep $cmsRunArguments\n");
print(SUBMIT "transfer_input_files = ");
for ($jobNum=0; $jobNum<=$#jobf-1; $jobNum++) {
print(SUBMIT "$jobf[$jobNum],");
}
print(SUBMIT "$jobf[$#jobf]\n");
print(SUBMIT "Queue\n");
}
}
close(SUBMIT);
sub specializeCfg($$@) {
my ($inp, $index, @files)=@_;
$stub2=$jobBase;
$stub2.=sprintf("_%03d",$index);
$mycfg="$prodSpace/$jobBase/cfg/".$stub2."_cfg.py";
print " $inp $index --> $stub2 ($mycfg) \n";
#print "$inp $text\n";
open(INP,$inp);
open(OUTP,">$mycfg");
$sector=0;
$had2=0;
$had3=0;
while(<INP>) {
if (/TFileService/) {
$sector=2;
$had2=1;
}
if (/PoolOutputModule/) {
$sector=3;
$had3=1;
}
if (/[.]Source/) {
$sector=1;
}
if (/rivetAnalyzer[.]OutputFile/) {
$sector=4;
}
# TFile Service Block
if ($sector==2 && /^[^\#]*fileName\s*=/) {
if ($had3==1) {
$fname=$stub2."-hist.root";
} else {
$fname=$stub2.".root";
}
unlink($fname);
print OUTP " fileName = cms.string(\"$fname\"),\n";
# PoolOutputModule Block
} elsif ($sector==3 && /^[^\#]*fileName\s*=/) {
if ($had2==1) {
$fname="$prodSpace/$jobBase/".$stub2."-pool.root";
} else {
$fname=$stub2.".root";
}
unlink($fname);
print OUTP " fileName = cms.untracked.string(\"$fname\"),\n";
# *Source Block (PoolSource, etc.)
} elsif ($sector==4 && /^[^\#]*rivetAnalyzer[.]OutputFile\s*=/) {
$fname="$prodSpace/$jobBase/".$stub2.".yoda";
unlink($fname);
print OUTP "process.rivetAnalyzer.OutputFile = cms.string(\"$fname\")\n";
# PoolOutputModule Block
} elsif ($sector==1 && /^[^\#]*fileNames\s*=/) {
print OUTP " fileNames=cms.untracked.vstring(\n";
for ($qq=0; $qq<=$#files; $qq++) {
$storefile=$files[$qq];
if ($storefile=~/store/) {
if ($use_xrootd) {
$storefile=~s|.*/store|root://cmsxrootd.fnal.gov//store|;
} else {
$storefile=~s|.*/store|/store|;
}
} else {
my($filename, $dirs, $suffix) = fileparse($storefile);
$storefile="file:".$filename;
}
print OUTP " '".$storefile."'";
print OUTP "," if ($qq!=$#files);
print OUTP "\n";
}
print OUTP " )\n";
} else {
print OUTP;
}
$depth++ if (/\{/ && $sector!=0);
if (/\}/ && $sector!=0) {
$depth--;
$sector=0 if ($depth==0);
}
# printf("%d %d %s",$sector,$depth,$_);
}
close(OUTP);
close(INP);
return $mycfg;
}