This repository has been archived by the owner on May 8, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 0
/
nbc_ibarrier.c
165 lines (140 loc) · 5.4 KB
/
nbc_ibarrier.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
/*
* Copyright (c) 2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2006 The Technical University of Chemnitz. All
* rights reserved.
*
* Author(s): Torsten Hoefler <htor@cs.indiana.edu>
*
*/
#include "nbc_internal.h"
#ifdef USE_DCMF
#include <dcmf_globalcollectives.h>
static int initialized=0;
static void cbfunc(void *clientdata, DCMF_Error_t *error) {
//printf("in callback!\n");
*(unsigned*)clientdata=2;
}
static DCMF_Protocol_t barr_reg;
static DCMF_Request_t barr_req;
static void init_dcmf_barrier() {
int r;
MPI_Comm_rank(MPI_COMM_WORLD, &r);
if(!r) printf("# initializing fast DCMF barrier\n");
DCMF_GlobalBarrier_Configuration_t barrier_config;
barrier_config.protocol = DCMF_GI_GLOBALBARRIER_PROTOCOL;
DCMF_GlobalBarrier_register(&barr_reg, &barrier_config);
initialized=1;
}
#endif
/* Dissemination implementation of MPI_Ibarrier */
#ifdef HAVE_SYS_WEAK_ALIAS_PRAGMA
#pragma weak NBC_Ibarrier=PNBC_Ibarrier
#define NBC_Ibarrier PNBC_Ibarrier
#endif
int NBC_Ibarrier(MPI_Comm comm, NBC_Handle* handle) {
int round, rank, p, maxround, res, recvpeer, sendpeer;
NBC_Schedule *schedule;
res = NBC_Init_handle(handle, comm);
if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; }
res = MPI_Comm_rank(comm, &rank);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; }
res = MPI_Comm_size(comm, &p);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_size() (%i)\n", res); return res; }
#ifdef USE_DCMF
int ws, s;
MPI_Comm_size(comm, &s);
MPI_Comm_size(MPI_COMM_WORLD, &ws);
if(s != ws) {
printf("DCMF only works on MPI_COMM_WORLD (or dups of it) for now -- fallback needs to be implemented :-)\n");
return NBC_NOT_IMPLEMENTED;
}
if(!initialized) init_dcmf_barrier();
handle->dcmf_hndl = (NBC_DCMF_Handle*)malloc(sizeof(NBC_DCMF_Handle));
handle->dcmf_hndl->done=0;
handle->dcmf_hndl->type=DCMF_TYPE_BARRIER;
DCMF_Callback_t callback={ cbfunc, &handle->dcmf_hndl->done };
//printf("[%i] LibNBC starting barrier\n", rank);
DCMF_GlobalBarrier(&barr_reg, &barr_req, callback);
//printf("[%i] LibNBC after barrier\n", rank);
#else
handle->tmpbuf=(void*)malloc(2*sizeof(char));
#ifdef NBC_CACHE_SCHEDULE
/* there only one argument set per communicator -> hang it directly at
* the tree-position, NBC_Dict_size[...] is 0 for not initialized and
* 1 for initialized. NBC_Dict[...] is a pointer to the schedule in
* this case */
if(handle->comminfo->NBC_Dict_size[NBC_BARRIER] == 0) {
/* we did not init it yet */
#endif
schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule));
if (NULL == schedule) { printf("Error in malloc()\n"); return res; }
round = -1;
res = NBC_Sched_create(schedule);
if(res != NBC_OK) { printf("Error in NBC_Sched_create (%i)\n", res); return res; }
maxround = (int)ceil((log(p)/LOG2)-1);
do {
round++;
sendpeer = (rank + (1<<round)) % p;
/* add p because modulo does not work with negative values */
recvpeer = ((rank - (1<<round))+p) % p;
/* send msg to sendpeer */
res = NBC_Sched_send((void*)0, true, 1, MPI_BYTE, sendpeer, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
/* recv msg from recvpeer */
res = NBC_Sched_recv((void*)1, true, 1, MPI_BYTE, recvpeer, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
/* end communication round */
if(round < maxround){
res = NBC_Sched_barrier(schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_barrier() (%i)\n", res); return res; }
}
} while (round < maxround);
res = NBC_Sched_commit(schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_commit() (%i)\n", res); return res; }
#ifdef NBC_CACHE_SCHEDULE
/* add it */
handle->comminfo->NBC_Dict[NBC_BARRIER] = (hb_tree*)schedule;
handle->comminfo->NBC_Dict_size[NBC_BARRIER] = 1;
} else {
/* we found it */
schedule = (NBC_Schedule*)handle->comminfo->NBC_Dict[NBC_BARRIER];
}
#endif
res = NBC_Start(handle, schedule);
if (NBC_OK != res) { printf("Error in NBC_Start() (%i)\n", res); return res; }
#endif
return NBC_OK;
}
#ifdef __cplusplus
extern "C" {
#endif
/* Fortran bindings */
#ifdef HAVE_SYS_WEAK_ALIAS_PRAGMA
NBC_F77_ALLFUNC_(nbc_ibarrier,NBC_IBARRIER,(int *fcomm, int *fhandle, int *ierr));
#pragma weak NBC_IBARRIER = nbc_ibarrier_f
#pragma weak nbc_ibarrier = nbc_ibarrier_f
#pragma weak nbc_ibarrier_ = nbc_ibarrier_f
#pragma weak nbc_ibarrier__ = nbc_ibarrier_f
#pragma weak PNBC_IBARRIER = nbc_ibarrier_f
#pragma weak pnbc_ibarrier = nbc_ibarrier_f
#pragma weak pnbc_ibarrier_ = nbc_ibarrier_f
#pragma weak pnbc_ibarrier__ = nbc_ibarrier_f
void nbc_ibarrier_f(int *fcomm, int *fhandle, int *ierr) {
#else
void NBC_F77_FUNC_(nbc_ibarrier,NBC_IBARRIER)(int *fcomm, int *fhandle, int *ierr);
void NBC_F77_FUNC_(nbc_ibarrier,NBC_IBARRIER)(int *fcomm, int *fhandle, int *ierr) {
#endif
MPI_Comm comm;
NBC_Handle *handle;
/* this is the only MPI-2 we need :-( */
comm = MPI_Comm_f2c(*fcomm);
/* create a new handle in handle table */
NBC_Create_fortran_handle(fhandle, &handle);
/* call NBC function */
*ierr = NBC_Ibarrier(comm, handle);
}
#ifdef __cplusplus
}
#endif