This repository has been archived by the owner on May 8, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 0
/
nbc_ibcast.c
287 lines (252 loc) · 9.87 KB
/
nbc_ibcast.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
/*
* Copyright (c) 2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2006 The Technical University of Chemnitz. All
* rights reserved.
*
* Author(s): Torsten Hoefler <htor@cs.indiana.edu>
*
*/
#include "nbc_internal.h"
static __inline__ int bcast_sched_binomial(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count, MPI_Datatype datatype);
static __inline__ int bcast_sched_linear(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count, MPI_Datatype datatype);
static __inline__ int bcast_sched_chain(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count, MPI_Datatype datatype, int fragsize, int size);
#ifdef NBC_CACHE_SCHEDULE
/* tree comparison function for schedule cache */
int NBC_Bcast_args_compare(NBC_Bcast_args *a, NBC_Bcast_args *b, void *param) {
if( (a->buffer == b->buffer) &&
(a->count == b->count) &&
(a->datatype == b->datatype) &&
(a->root == b->root) ) {
return 0;
}
if( a->buffer < b->buffer ) {
return -1;
}
return +1;
}
#endif
#ifdef HAVE_SYS_WEAK_ALIAS_PRAGMA
#pragma weak NBC_Ibcast=PNBC_Ibcast
#define NBC_Ibcast PNBC_Ibcast
#endif
int NBC_Ibcast(void *buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm, NBC_Handle* handle) {
int rank, p, res, size, segsize;
NBC_Schedule *schedule;
#ifdef NBC_CACHE_SCHEDULE
NBC_Bcast_args *args, *found, search;
#endif
enum { NBC_BCAST_LINEAR, NBC_BCAST_BINOMIAL, NBC_BCAST_CHAIN } alg;
res = NBC_Init_handle(handle, comm);
if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; }
res = MPI_Comm_rank(comm, &rank);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; }
res = MPI_Comm_size(comm, &p);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; }
res = MPI_Type_size(datatype, &size);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_size() (%i)\n", res); return res; }
segsize = 16384;
/* algorithm selection */
if(p <= 4) {
alg = NBC_BCAST_LINEAR;
} else if(size*count < 65536) {
alg = NBC_BCAST_BINOMIAL;
} else if(size*count < 524288) {
alg = NBC_BCAST_CHAIN;
segsize = 16384/2;
} else {
alg = NBC_BCAST_CHAIN;
segsize = 65536/2;
}
handle->tmpbuf=NULL;
#ifdef NBC_CACHE_SCHEDULE
/* search schedule in communicator specific tree */
search.buffer=buffer;
search.count=count;
search.datatype=datatype;
search.root=root;
found = (NBC_Bcast_args*)hb_tree_search((hb_tree*)handle->comminfo->NBC_Dict[NBC_BCAST], &search);
if(found == NULL) {
#endif
schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule));
res = NBC_Sched_create(schedule);
if(res != NBC_OK) { printf("Error in NBC_Sched_create, res = %i\n", res); return res; }
switch(alg) {
case NBC_BCAST_LINEAR:
res = bcast_sched_linear(rank, p, root, schedule, buffer, count, datatype);
break;
case NBC_BCAST_BINOMIAL:
res = bcast_sched_binomial(rank, p, root, schedule, buffer, count, datatype);
break;
case NBC_BCAST_CHAIN:
res = bcast_sched_chain(rank, p, root, schedule, buffer, count, datatype, segsize, size);
break;
}
if (NBC_OK != res) { printf("Error in Schedule creation() (%i)\n", res); return res; }
res = NBC_Sched_commit(schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_commit() (%i)\n", res); return res; }
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Bcast_args*)malloc(sizeof(NBC_Bcast_args));
args->buffer=buffer;
args->count=count;
args->datatype=datatype;
args->root=root;
args->schedule=schedule;
res = hb_tree_insert ((hb_tree*)handle->comminfo->NBC_Dict[NBC_BCAST], args, args, 0);
if(res != 0) printf("error in dict_insert() (%i)\n", res);
/* increase number of elements for A2A */
if(++handle->comminfo->NBC_Dict_size[NBC_BCAST] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe((hb_tree*)handle->comminfo->NBC_Dict[NBC_BCAST], &handle->comminfo->NBC_Dict_size[NBC_BCAST]);
}
} else {
/* found schedule */
schedule=found->schedule;
}
#endif
res = NBC_Start(handle, schedule);
if (NBC_OK != res) { printf("Error in NBC_Start() (%i)\n", res); return res; }
return NBC_OK;
}
/* better binomial bcast
* working principle:
* - each node gets a virtual rank vrank
* - the 'root' node get vrank 0
* - node 0 gets the vrank of the 'root'
* - all other ranks stay identical (they do not matter)
*
* Algorithm:
* - each node with vrank > 2^r and vrank < 2^r+1 receives from node
* vrank - 2^r (vrank=1 receives from 0, vrank 0 receives never)
* - each node sends each round r to node vrank + 2^r
* - a node stops to send if 2^r > commsize
*/
#define RANK2VRANK(rank, vrank, root) \
{ \
vrank = rank; \
if (rank == 0) vrank = root; \
if (rank == root) vrank = 0; \
}
#define VRANK2RANK(rank, vrank, root) \
{ \
rank = vrank; \
if (vrank == 0) rank = root; \
if (vrank == root) rank = 0; \
}
static __inline__ int bcast_sched_binomial(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count, MPI_Datatype datatype) {
int maxr, vrank, peer, r, res;
maxr = (int)ceil((log(p)/LOG2));
RANK2VRANK(rank, vrank, root);
/* receive from the right hosts */
if(vrank != 0) {
for(r=0; r<maxr; r++) {
if((vrank >= (1<<r)) && (vrank < (1<<(r+1)))) {
VRANK2RANK(peer, vrank-(1<<r), root);
res = NBC_Sched_recv(buffer, false, count, datatype, peer, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
}
}
res = NBC_Sched_barrier(schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_barrier() (%i)\n", res); return res; }
}
/* now send to the right hosts */
for(r=0; r<maxr; r++) {
if(((vrank + (1<<r) < p) && (vrank < (1<<r))) || (vrank == 0)) {
VRANK2RANK(peer, vrank+(1<<r), root);
res = NBC_Sched_send(buffer, false, count, datatype, peer, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
}
}
return NBC_OK;
}
/* simple linear MPI_Ibcast */
static __inline__ int bcast_sched_linear(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count, MPI_Datatype datatype) {
int peer, res;
/* send to all others */
if(rank == root) {
for (peer=0; peer<p;peer++) {
if(peer != root) {
/* send msg to peer */
res = NBC_Sched_send(buffer, false, count, datatype, peer, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
}
}
} else {
/* recv msg from root */
res = NBC_Sched_recv(buffer, false, count, datatype, root, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
}
return NBC_OK;
}
/* simple chained MPI_Ibcast */
static __inline__ int bcast_sched_chain(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count, MPI_Datatype datatype, int fragsize, int size) {
int res, vrank, rpeer, speer, numfrag, fragnum, fragcount, thiscount;
MPI_Aint ext;
char *buf;
RANK2VRANK(rank, vrank, root);
VRANK2RANK(rpeer, vrank-1, root);
VRANK2RANK(speer, vrank+1, root);
res = MPI_Type_extent(datatype, &ext);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
if(count == 0) return NBC_OK;
numfrag = count*size/fragsize;
if((count*size)%fragsize != 0) numfrag++;
fragcount = count/numfrag;
/*if(!rank) printf("numfrag: %i, count: %i, size: %i, fragcount: %i\n", numfrag, count, size, fragcount);*/
for(fragnum = 0; fragnum < numfrag; fragnum++) {
buf = (char*)buffer+fragnum*fragcount*ext;
thiscount = fragcount;
if(fragnum == numfrag-1) {
/* last fragment may not be full */
thiscount = count-fragcount*fragnum;
}
/* root does not receive */
if(vrank != 0) {
res = NBC_Sched_recv(buf, false, thiscount, datatype, rpeer, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
res = NBC_Sched_barrier(schedule);
}
/* last rank does not send */
if(vrank != p-1) {
res = NBC_Sched_send(buf, false, thiscount, datatype, speer, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
/* this barrier here seems awaward but isn't!!!! */
if(vrank == 0) res = NBC_Sched_barrier(schedule);
}
}
return NBC_OK;
}
#ifdef __cplusplus
extern "C" {
#endif
/* Fortran bindings */
#ifdef HAVE_SYS_WEAK_ALIAS_PRAGMA
NBC_F77_ALLFUNC_(nbc_ibcast,NBC_IBCAST,(void *buf, int *count, int *datatype, int *root, int *fcomm, int *fhandle, int *ierr));
#pragma weak NBC_IBCAST = nbc_ibcast_f
#pragma weak nbc_ibcast = nbc_ibcast_f
#pragma weak nbc_ibcast_ = nbc_ibcast_f
#pragma weak nbc_ibcast__ = nbc_ibcast_f
#pragma weak PNBC_IBCAST = nbc_ibcast_f
#pragma weak pnbc_ibcast = nbc_ibcast_f
#pragma weak pnbc_ibcast_ = nbc_ibcast_f
#pragma weak pnbc_ibcast__ = nbc_ibcast_f
void nbc_ibcast_f(void *buf, int *count, int *datatype, int *root, int *fcomm, int *fhandle, int *ierr) {
#else
void NBC_F77_FUNC_(nbc_ibcast,NBC_IBCAST)(void *buf, int *count, int *datatype, int *root, int *fcomm, int *fhandle, int *ierr);
void NBC_F77_FUNC_(nbc_ibcast,NBC_IBCAST)(void *buf, int *count, int *datatype, int *root, int *fcomm, int *fhandle, int *ierr) {
#endif
MPI_Datatype dtype;
MPI_Comm comm;
NBC_Handle *handle;
/* this is the only MPI-2 we need :-( */
dtype = MPI_Type_f2c(*datatype);
comm = MPI_Comm_f2c(*fcomm);
/* create a new handle in handle table */
NBC_Create_fortran_handle(fhandle, &handle);
/* call NBC function */
*ierr = NBC_Ibcast(buf, *count, dtype, *root, comm, handle);
}
#ifdef __cplusplus
}
#endif