1 friday, november 10, 2006 “ programs for sale: fast, reliable, cheap: choose two.” -anonymous

1

Friday, November 10, 2006

“Programs for sale: Fast, Reliable, Cheap:

choose two.”

- Anonymous

2

#pragma omp parallel for

for(i=1; i<=n; i++)

{

temp = 2.0*a[i];

a[i] = temp;

b[i] = c[i]/temp;

}

3

When the private clause is encountered, a separate memory location is allocated for each specified variable on each thread. The value of the variable is not initialized; a location is simply set aside for each thread. Private variables must be initialized within the loop.

#pragma omp parallel for private(temp)

for(i=1; i<=n; i++)

{

temp = 2.0*a[i];

a[i] = temp;

b[i] = c[i]/temp;

}

4


for(i=1; i<=n; i++)

{

temp = 2.0*a[i];

a[i] = temp; b[i] = c[i]/temp;

}


{

for(i=1; i<=n; i++)

{

temp = 2.0*a[i];

a[i] = temp; b[i] = c[i]/temp;

}

} Intel compiler gives error here.

5

Note that the loop index, i , is always private no matter what the default.

#pragma omp parallel for default(private) shared(n,a,b,c) for(i=1; i<=n; i++){

temp = 2.0*a[i]; a[i] = temp; b[i] = c[i]/temp;

}

6

j = jstart;

#pragma omp parallel for private(j)

for(i=1; i<=n; i++){

if(i == 1 || i == n)

j = j + 1;

a[i] = a[i] + j;

}

What is wrong here?

7

j = jstart;

#pragma omp parallel for firstprivate(j)

for(i=1; i<=n; i++){

if(i == 1 || i == n)

j = j + 1;

a[i] = a[i] + j;

}

8

#pragma omp parallel for lastprivate(x)

for(i=1; i<=n; i++)

{

x = sin( pi * dx * (float)i );

a[i] = exp(x);

}

lastx = x;

parallel for pragma may have both firstprivate and lastprivate clauses and they may have variables in common.

9

Environment variable OMP_NUM_THREADS

In bash:

export OMP_NUM_THREADS=4

10

#include <stdio.h>#include <omp.h>int main(void){ int i,tid, numthreads, numprocs, size=10; numthreads = omp_get_num_threads(); numprocs = omp_get_num_procs(); printf("Numthreads before for-loop is=%d numprocs=%d\

n", numthreads, numprocs);

#pragma omp parallel for private(tid) schedule(static,2)

for(i=0; i<size; i++){ numthreads = omp_get_num_threads(); tid = omp_get_thread_num(); printf("Numthreads after for-loop is=%d\n",

numthreads); printf("I am thread=%d, I have iteration=%d\n", tid,

i); } return 0;}

11

omp_get_thread_numReturns the thread rank in a parallel region.The rank of threads ranges from 0 to

omp_get_num_threads() - 1.

12

To compile:

icc -openmp myprog.c -o myprog

Note: Logout of past sessions for compiler environment settings to take effect.

13

Difference between front-end and compute nodes.

14

Numthreads before for-loop is=1 numprocs=4Numthreads after for-loop is=4I am thread=0, I have iteration=0Numthreads after for-loop is=4I am thread=0, I have iteration=1Numthreads after for-loop is=4I am thread=0, I have iteration=8Numthreads after for-loop is=4I am thread=0, I have iteration=9Numthreads after for-loop is=4I am thread=1, I have iteration=2Numthreads after for-loop is=4I am thread=1, I have iteration=3Numthreads after for-loop is=4I am thread=2, I have iteration=4Numthreads after for-loop is=4I am thread=2, I have iteration=5Numthreads after for-loop is=4I am thread=3, I have iteration=6Numthreads after for-loop is=4I am thread=3, I have iteration=7

15

void omp_set_num_threads (int t)

It uses parameter value to set the number of threads to be active in parallel sections of code.

We have the ability to tailor the level of parallelism. call omp_set_num_threads prior to the beginning of a

parallel region for it to take effect; The result is undefined if this subroutine is called

within a parallel region.

16

#include <stdio.h>#include <omp.h>int main(void){ int i,tid, size=10; omp_set_num_threads(4); #pragma omp parallel for private(tid)

schedule(static,2) for(i=0; i<size; i++){ tid = omp_get_thread_num(); printf("I am thread=%d, I have

iteration=%d\n", tid, i); }

return 0;}

17

I am thread=0, I have iteration=0I am thread=0, I have iteration=1I am thread=0, I have iteration=8I am thread=0, I have iteration=9I am thread=1, I have iteration=2I am thread=1, I have iteration=3I am thread=2, I have iteration=4I am thread=2, I have iteration=5I am thread=3, I have iteration=6I am thread=3, I have iteration=7

18

#include <stdio.h>#include <omp.h>

int main(void){ int i,tid, size=10; #pragma omp parallel for private(tid) schedule(static,2) num_threads(4)

for(i=0; i<size; i++){ tid = omp_get_thread_num(); printf("I am thread=%d, I have iteration=%d\n", tid, i);

}

return 0;}

19

What is wrong here?

double area, pi, x;int i,n;//....area = 0.0;#pragma omp parallel for private(x) for (i=0; i<n; i++){ x=(i+0.5)/n; area+=4.0/(1.0+x*x); } pi=area/n;

20

What is wrong here?

double area, pi, x;int i,n;//....area = 0.0;#pragma omp parallel for private(x) for (i=0; i<n; i++){ x=(i+0.5)/n; area+=4.0/(1.0+x*x);// Race Condition

} pi=area/n;

21

double area, pi, x;int i,n;//....area = 0.0;#pragma omp parallel for private(x) for (i=0; i<n; i++){ x=(i+0.5)/n;#pragma omp critical area+=4.0/(1.0+x*x); } pi=area/n;

22

double area, pi, x;int i,n;//....area = 0.0;#pragma omp parallel for private(x) for (i=0; i<n; i++){ x=(i+0.5)/n;#pragma omp critical area+=4.0/(1.0+x*x); } pi=area/n;

This can affect speedup.

23

for(i=1; i<=n; i++){ sum = sum + a[i];

}

24

#pragma omp parallel for reduction(+:sum) for(i=1; i<=n; i++){

sum = sum + a[i]; }

25

Different reduction operations and initial values

+ 0

- 0

* 1

& all bits 1

| 0

^ 0

&& 1

|| 0

26

double area, pi, x;int i,n;//....area = 0.0;#pragma omp parallel for private(x) reduction (+:area)

for (i=0; i<n; i++){ x=(i+0.5)/n; area+=4.0/(1.0+x*x); } pi=area/n;

27

Conditional execution of loops

double area, pi, x;int i,n;//....area = 0.0;#pragma omp parallel for private(x) reduction (+:area) if(n>5000)

for (i=0; i<n; i++){ x=(i+0.5)/n; area+=4.0/(1.0+x*x); } pi=area/n;

28

for(i=1; i<=n; i++)

{ myval = do_lots_of_work(i);

printf("%d %d\n", i, myval);

}

29

#pragma omp parallel for private(myval) orderedfor(i=1; i<=n; i++){

myval = do_lots_of_work(i); #pragma omp ordered {

printf("%d %d\n", i, myval);}

}

Note: The opening curly brace may not appear on the same line as the ordered directive.

30

for (i=0; i<BLOCK_SIZE(id,p,n); i++)

for(j=0; j<n; j++)a[i][j]=MIN(a[i][j], a[i][k]+tmp[j]);

If we parallelize inner loop what would happen?

31

for (i=0; i<BLOCK_SIZE(id,p,n); i++)for(j=0; j<n; j++)

a[i][j]=MIN(a[i][j], a[i][k]+tmp[j]);

fork/join overhead for every iteration of outer loop.

If we parallelize outer loop we only incur fork/join overhead once.

32




33




Problem here!

34

#pragma omp parallel for private(j)



35

Scheduling loops n is the number of iterations and t is the number of threads

schedule (static) Static allocation of about n/t contiguous iterations to

each thread.

schedule (static, C) Allocation of chunks to tasks. Each chunk contains C

contiguous iterations. Assigned in round-robin fashion.

schedule (dynamic) Iterations are dynamically allocated, one at a time, to

threads.

36

Scheduling loopsschedule (dynamic, C) Iterations are dynamically

allocated, C iterations at a time, to threads.schedule (guided, C)

Also called guided self scheduling. The first chunk is an implementation dependent size and each successive chunk is a fixed fraction of preceding chunk until the minimum chunk size C is reached.

schedule (guided) Same as above, C is taken to be 1schedule (runtime) The schedule type is chosen at

runtime based on value of environment variable OMP_SCHEDULEe.g. export OMP_SCHEDULE=“static,1”

37

In OpenMP, there are two main approaches for assigning work to threads:

Loop-levelParallel regions

In the first approach, loop-level, individual loops are parallelized with each thread being assigned a unique range of the loop index.

In parallel regions, any sections of the code can be parallelized, not just loops. The work within the parallel regions is

explicitly distributed among the threads using the unique identifier assigned to each thread.

This can be done by using if statements, e.g., if(mythreadid == 0) …

38

In the loop-level approach, execution starts on a single thread. Then, when a parallel loop is encountered, multiple threads are spawned. When the parallel loop is finished, the extra threads are discarded and the execution is once again serial until the next parallel loop.

In the parallel-regions approach, multiple threads are maintained, irrespective of whether or not loops are encountered.

39

When using the parallel directive that the entire region of code between within braces will be duplicated on all threads.

This allows more flexibility than restricting and we can parallelize code in a manner much like that used with MPI.

40


for(i=1; i<=maxi; i++)

{

a(i) = b(i);

}

#pragma omp parallel

#pragma omp for

for(i=1; i<=maxi; i++)

{

a(i) = b(i);

}

41

#include <stdio.h>#include <omp.h>int main(void){ int i,tid, numthreads, size=10; #pragma omp parallel for private(tid) schedule(static,2)

for(i=0; i<size; i++){ numthreads = omp_get_num_threads(); tid = omp_get_thread_num(); printf("Numthreads after for-loop is=%d\n", numthreads);

printf("I am thread=%d, I have iteration=%d\n", tid, i);

}

return 0;}

42

#include <stdio.h>#include <omp.h>int main(void){ int i,tid, numthreads, size=10; #pragma omp parallel private(tid) { numthreads = omp_get_num_threads(); tid = omp_get_thread_num(); printf("Numthreads after for-loop is=%d\n", numthreads);

#pragma omp for schedule(static,2) for(i=0; i<size; i++){ printf("I am thread=%d, I have iteration=%d\n", tid, i);

} } return 0;}

43

Numthreads after for-loop is=4I am thread=0, I have iteration=0I am thread=0, I have iteration=1I am thread=0, I have iteration=8I am thread=0, I have iteration=9Numthreads after for-loop is=4I am thread=1, I have iteration=2I am thread=1, I have iteration=3Numthreads after for-loop is=4Numthreads after for-loop is=4I am thread=3, I have iteration=6I am thread=3, I have iteration=7I am thread=2, I have iteration=4I am thread=2, I have iteration=5

44

#pragma omp parallel { #pragma omp for for(i=1; i<=maxi; i++){

a[i] = b[i]; } #pragma omp for for(j=1; j<=maxj; j++){

c[j] = d[j]; }

}

•Advantages when there are multiple loops

45

There is an implied barrier at the end of a loop with for directive.

If a barrier is not desired, the nowait clause can be used.

46

#pragma omp parallel {

#pragma omp for nowait for(i=1; i<=maxi; i++){

a[i] = b[i]; } #pragma omp for for(j=1; j<=maxj; j++){

c[j] = d[j]; }

}

47

In the loop-level approach, domain decomposition is performed automatically by distributing loop indices among the threads.

In the parallel regions approach, domain decomposition is performed manually.

48

#pragma omp parallel private(myid,istart,iend,nthreads,nper)

{ nthreads = omp_get_num_threads(); nper = imax/nthreads; myid = omp_get_thread_num(); istart = myid*nper + 1;iend = istart + nper - 1; do_work(istart,iend); for(i=istart; i<=iend; i++)

a(i) = b(i)*c(i) }

The size of the arrays is imax, and nper is the number of indices per thread.

49

Master /slave programming in message passing model.

The shared memory model allows each thread to access the list of tasks, so there is no need for a separate master thread.

50

pseudo-codeint main(){

struct task_struct task_ptr;…task_ptr=get_next_task();while(task_ptr != NULL){

complete_task(task_ptr);task_ptr = get_next_task()

}…}

51

int main(){struct task_struct task_ptr;…

#pragma omp parallel private (task_ptr) {

task_ptr=get_next_task();while(task_ptr != NULL){

complete_task(task_ptr);task_ptr = get_next_task()

} }…}

task_struct get_next_task() {#pragma omp critical{

// here shared linked-list is modified.}

52

SELF-TEST

Example 7.11Example 7.12

53

SELF TEST: Matrix Multiplication

/* static scheduling of matrix multiplication loops */

#pragma omp parallel default(private) shared (a, b, c, dim) num_threads(4)#pragma omp for schedule(static)for (i = 0; i < dim; i++) {

for (j = 0; j < dim; j++) {c(i,j) = 0;for (k = 0; k < dim; k++) {

c(i,j) += a(i, k) * b(k, j);}

}}

54

v=alpha();

w=beta();

x=gamma(v,w);

y=delta();

epsilon(x,y);

alpha beta

gamma

epsilon

delta

55

parallel sections pragma precedes a block of k blocks of code that may be executed concurrently by k threads.

section pragma is within the encompassing parallel sections block and identifies individual blocks in it.

There is an implied barrier at end of sections.

56

#pragma omp parallel sections{#pragma omp section /*optional pragma*/

v=alpha();#pragma omp section

w=beta();#pragma omp section

y=delta();}x=gamma(v,w);epsilon(x,y);

alpha beta

gamma

epsilon

delta

57


#pragma omp sections {

#pragma omp section /*optional pragma*/v=alpha();

#pragma omp sectionw=beta();

}#pragma omp sections

{#pragma omp section /*optional pragma*/

x=gamma(v,w);#pragma omp section

y=delta(); }}epsilon(x,y);

alpha beta

gamma

epsilon

delta

58


#pragma omp sections {

#pragma omp section /*optional pragma*/v=alpha();

#pragma omp sectionw=beta();

}#pragma omp sections

{#pragma omp section /*optional pragma*/

x=gamma(v,w);#pragma omp section

y=delta(); }}epsilon(x,y);

alpha beta

gamma

epsilon

delta

Better if only two processors are available.

59

#pragma omp single For example, for printing error messages

#pragma omp master

omp_set_dynamicOMP_DYNAMIC

1 friday, november 10, 2006 “ programs for sale: fast, reliable, cheap: choose two.” -anonymous

Documents

anonymous slide

pragma omp parallel