Parallel programmingin PythonJonas Juselius <jonas.juselius@uit.no>

HPC@UiT

About writing parallel programs

• It's hard, hard, hard

1

Parallel paradigms

• Trivial parallelism

• Multi-threading and shared memory

• Message passing

2

3

Parallel Python (SMP) 1 #!/usr/bin/env python 2 import pp 3 import math 4 from scipy.integrate import quad 5 import scipy.integrate 6 7 n = 4 8 ival = (-20.0, 20.0) 9 step = sum(map(abs, ival))/n10 11 server = pp.Server(n)12 13 def f(x): 14 return math.exp(-0.1*x**2)15 16 def integrate(f, a, b): 17 '''Wrapper for quad so name resolution works'''18 return scipy.integrate.quad(f, a, b)19 20 foo = lambda i: (ival[0] + i * step, ival[0] + (i + 1) * step)

4

21 view = [ foo(i) for i in range(n)]22 23 p = []24 for i in range(n):25 a, b = view[i]26 p.append(server.submit(integrate, (f, a, b), (f,), ('math',27 'scipy.integrate')))28 29 I = 0.030 for i in range(n):31 r = p[i]()32 I += r[0]33 34 a, b = ival35 print I, quad(f, a, b)[0]36 print server.print_stats()

5

6

Parallel Python (Cluster) 1 #!/usr/bin/env python 2 import pp 3 import math 4 from scipy.integrate import quad 5 import scipy.integrate 6 7 n = 4 8 ival = (-20.0, 20.0) 9 step = sum(map(abs, ival))/n10 11 # must first start ppserver.py on all remote nodes!12 nodes = [ "localhost" for i in range(n) ]13 server = pp.Server(n, ppservers=tuple(nodes))14 15 def f(x): 16 return math.exp(-0.1*x**2)17 18 def integrate(f, a, b): 19 '''Wrapper for quad so name resolution works'''20 return scipy.integrate.quad(f, a, b)

7

21 22 foo = lambda i: (ival[0] + i * step, ival[0] + (i + 1) * step)23 view = [ foo(i) for i in range(n)]24 25 p = []26 for i in range(n):27 a, b = view[i]28 p.append(server.submit(integrate, (f, a, b), (f,), ('math',29 'scipy.integrate')))30 31 I = 0.032 for i in range(n):33 r = p[i]()34 I += r[0]35 36 a, b = ival37 print I, quad(f, a, b)[0]38 print server.print_stats()

8

MPI: Point-to-point

1 #!/usr/bin/env python 2 from mpi4py import MPI 3 import numpy 4 5 comm = MPI.COMM_WORLD 6 rank = comm.Get_rank() 7 8 # pass explicit MPI datatypes 9 if rank == 0:10 data = [ "foo" for i in range(5) ] 11 comm.send(data, dest=1, tag=77)12 elif rank == 1:13 data = comm.recv(source=0, tag=77)14 print data

9

10

MPI: Point-to-point (numpy)

1 #!/usr/bin/env python 2 from mpi4py import MPI 3 import numpy 4 5 comm = MPI.COMM_WORLD 6 rank = comm.Get_rank() 7 8 # pass explicit MPI datatypes 9 if rank == 0:10 data = numpy.arange(100, dtype='i')11 comm.Send([data, MPI.INT], dest=1, tag=77)12 elif rank == 1:13 data = numpy.empty(100, dtype='i')14 comm.Recv([data, MPI.INT], source=0, tag=77)15 print data16

11

17 # automatic MPI datatype discovery18 if rank == 0:19 data = numpy.arange(100, dtype=numpy.float64)20 comm.Send(data, dest=1, tag=13)21 elif rank == 1:22 data = numpy.empty(100, dtype=numpy.float64)23 comm.Recv(data, source=0, tag=13)24 print data

12

13

MPI: Scatter and gather

1 #!/usr/bin/env python 2 # 3 # Simple scatter-gather example using mpi4py 4 # 5 # $ mpirun -np 4 python ./foo.py 6 # 7 import numpy as np 8 from mpi4py import MPI 9 10 comm = MPI.COMM_WORLD11 rank = comm.Get_rank()12 size = comm.Get_size()13 14 if rank == 0:15 data = [(i+1)**2 for i in xrange(size)]16 else:

14

17 data = None18 19 print 'foo ', rank, data20 21 data = comm.scatter(data, root=0)22 print 'rank=', rank, data23 data += 10024 data = comm.gather(data, root=0)25 26 print 'rank=', rank, data

15

16

MPI: Scatter and gather (numpy)

1 #!/usr/bin/env python 2 # 3 # Simple scatter-gather example using mpi4py 4 # 5 # $ mpirun -np 4 python ./foo.py 6 # 7 import numpy as np 8 from mpi4py import MPI 9 10 comm = MPI.COMM_WORLD11 rank = comm.Get_rank()12 size = comm.Get_size()13 14 if rank == 0:15 data = np.arange(0.0, 40.0)16 else:

17

17 data = np.array()18 19 print 'foo ', rank, data20 21 data = comm.Scatter([data, MPI.DOUBLE], root=0)22 #print 'rank=', rank, data23 #data = data + 10024 data = comm.Gather([data, MPI.DOUBLE], root=0)25 26 #print 'rank=', rank, data

18

19

MPI: Reduction 1 #!/usr/bin/env python 2 from mpi4py import MPI 3 import numpy as np 4 from scipy.integrate import quad 5 import math 6 7 comm = MPI.COMM_WORLD 8 size = comm.Get_size() 9 rank = comm.Get_rank()10 11 ival = (-20.0, 20.0)12 step = sum(map(abs, ival))/size13 14 view = None15 if rank == 0:16 foo = lambda i: (ival[0] + i * step, ival[0] + (i + 1) * step)17 view = [ foo(i) for i in range(size)]18 19 # silly: scatter is better, but utilizing rank is best20 view = comm.bcast(view, root=0)

20

21 22 f = lambda x: math.exp(-0.1*x**2)23 a, b = view[rank]24 I = quad(f, a, b)[0]25 26 if rank == 0:27 I = comm.reduce(0.0, I, op=MPI.SUM, root=0)28 print I, quad(f, -20.0, 20.0)[0]29 else:30 comm.reduce(I, None, op=MPI.SUM, root=0)

21