Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix Scaffold #339

Merged
merged 4 commits into from
Nov 6, 2023
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Update scaffold.py
  • Loading branch information
hua-zi authored Nov 3, 2023
commit cd2d681948a4ad3b9563bd3a95d0fe40e3cddbff
31 changes: 23 additions & 8 deletions fedlab/contrib/algorithm/scaffold.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ def downlink_package(self):

def setup_optim(self, lr):
self.lr = lr
self.global_c = torch.zeros_like(self.model_parameters)
# self.global_c = torch.zeros_like(self.model_parameters)
self.global_c = torch.zeros_like(self.model_grad_parameters)

def global_update(self, buffer):
# unpack
Expand Down Expand Up @@ -74,19 +75,33 @@ def train(self, id, model_parameters, global_c, train_loader):
self.optimizer.zero_grad()
loss.backward()

grad = self.model_gradients
# grad = self.model_gradients
grad = self.model_grads
grad = grad - self.cs[id] + global_c
idx = 0
for parameter in self._model.parameters():
layer_size = parameter.grad.numel()
shape = parameter.grad.shape
#parameter.grad = parameter.grad - self.cs[id][idx:idx + layer_size].view(parameter.grad.shape) + global_c[idx:idx + layer_size].view(parameter.grad.shape)
parameter.grad.data[:] = grad[idx:idx+layer_size].view(shape)[:]

parameters = self._model.parameters()
for p in self._model.state_dict().values():
if p.grad is None: # batchnorm have no grad
layer_size = p.numel()
else:
parameter = next(parameters)
layer_size = parameter.data.numel()
shape = parameter.grad.shape
parameter.grad.data[:] = grad[idx:idx+layer_size].view(shape)[:]
idx += layer_size

# for parameter in self._model.parameters():
# layer_size = parameter.grad.numel()
# shape = parameter.grad.shape
# #parameter.grad = parameter.grad - self.cs[id][idx:idx + layer_size].view(parameter.grad.shape) + global_c[idx:idx + layer_size].view(parameter.grad.shape)
# parameter.grad.data[:] = grad[idx:idx+layer_size].view(shape)[:]
# idx += layer_size
# print(idx)

self.optimizer.step()

dy = self.model_parameters - frz_model
dc = -1.0 / (self.epochs * len(train_loader) * self.lr) * dy - global_c
dc = -1.0 / (self.epochs * len(train_loader) * self.lr) * dy - global_c # ????
self.cs[id] += dc
return [dy, dc]