Run shell script in parallel using


#!/bin/env python
#Run shell scripts in parallel
#using pythong multiprocessing module
# Raju Konduru

import multiprocessing
import time
import os
import sys
import subprocess
from pprint import pprint

if len(sys.argv) == 4:
scriptToRun = sys.argv[1]
inputFile = sys.argv[2]
numOfProcess = int(sys.argv[3])
print type(numOfProcess)
print “Need 2 arguments 1st one input file 2nd one number of concurrent jobs”
print “Example sys.argv[0] myinput.csv 2”

inFile = open(inputFile,’r’)

def mp_worker(GRP):
print “Processs script”+scriptToRun+’ ‘+GRP,GRP)

def mp_handler():
p = multiprocessing.Pool(numOfProcess)
with open(inputFile,’r’) as source_file:
results =, source_file,numOfProcess)
print results

if __name__ == ‘__main__’:




Author: rajukv

Hadoop(BigData) Architect and Hadoop Security Architect can design and build hadoop system to meet various data science projects.

Leave a Reply

Fill in your details below or click an icon to log in: Logo

You are commenting using your account. Log Out /  Change )

Google photo

You are commenting using your Google account. Log Out /  Change )

Twitter picture

You are commenting using your Twitter account. Log Out /  Change )

Facebook photo

You are commenting using your Facebook account. Log Out /  Change )

Connecting to %s

This site uses Akismet to reduce spam. Learn how your comment data is processed.