Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[DA-4204] Adding Pagination to workbench/audit/workspace/snapshots API #3853

Open
wants to merge 4 commits into
base: devel
Choose a base branch
from
Open

[DA-4204] Adding Pagination to workbench/audit/workspace/snapshots API #3853

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5a9a3ab
Adding pagination to workbench/audit/workspace/snapshots API
vmoniqb Apr 5, 2024
5822af7
Decreasing MAX_MAX_RESULTS to Prevent API Timeout
vmoniqb Apr 9, 2024
e078434
Correcting unit test description
vmoniqb Apr 9, 2024
358c8b8
Applying MAX_MAX_RESULTS updates to Workbench API only
vmoniqb Apr 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion rdr_service/api/base_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@


DEFAULT_MAX_RESULTS = 100
MAX_MAX_RESULTS = 10000
MAX_MAX_RESULTS = 6000
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This update affects the Paricipant Summary API too. Is that intended, or should the code be updated to only set this as the max for the research workbench API's?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was intended, but after discussion in Slack, I will override the value instead to avoid any errors for awardees currently calling the PaarticipantSummary API with _count > 6000



def log_api_request(log: RequestsLog = None, model_obj=None):
Expand Down
17 changes: 16 additions & 1 deletion rdr_service/api/redcap_workbench_audit_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,16 @@ def __init__(self):

def get(self):
super(RedcapWorkbenchAuditApi, self).get()
return self.dao.workspace_dao.get_redcap_audit_workspaces(**self.get_filters)

# No pagination required, response always contains a single record
if self.get_filters.get('snapshot_id') is not None or self.get_filters.get('workspace_id') is not None:
return self.dao.workspace_dao.get_redcap_audit_workspaces(self.get_filters.get('snapshot_id'),
self.get_filters.get('workspace_id'))
# Pagination is required for last_snapshot_id and returning all snapshots
if self.get_filters.get('last_snapshot_id') is not None:
self.dao.workspace_dao.last_snapshot_id = self.get_filters.get('last_snapshot_id')
response = self._query("snapshotId")
return response

def _do_insert(self, m):
audit_records = super()._do_insert(m)
Expand All @@ -77,6 +86,12 @@ def _do_insert(self, m):
in_seconds=30, queue='resource-rebuild')
return audit_records

@classmethod
def _make_resource_url(cls, response_json, id_field, participant_id):
from rdr_service import main
url = main.api.url_for(cls, snapshot_id=response_json[0][id_field], _external=True)
return url

class RedcapResearcherAuditApi(BaseRedcapApi):
def __init__(self):
super().__init__()
Expand Down
266 changes: 156 additions & 110 deletions rdr_service/dao/workbench_dao.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,15 @@
WorkbenchResearcherAccessTierShortName, WorkbenchResearcherEthnicCategory, WorkbenchResearcherSexualOrientationV2, \
WorkbenchResearcherGenderIdentity, WorkbenchResearcherYesNoPreferNot, WorkbenchResearcherSexAtBirthV2,\
WorkbenchResearcherEducationV2
from rdr_service.query import Results
from rdr_service.services.system_utils import list_chunks


class WorkbenchWorkspaceDao(UpdatableDao):
def __init__(self):
super().__init__(WorkbenchWorkspaceApproved, order_by_ending=["id"])
self.is_backfill = False
self.last_snapshot_id = None
self.workspace_snapshot_dao = WorkbenchWorkspaceHistoryDao()

def get_id(self, obj):
Expand Down Expand Up @@ -319,12 +321,10 @@ def remove_workspace_by_workspace_id_with_session(session, workspace_id):

def get_redcap_audit_workspaces(
self,
last_snapshot_id=None,
snapshot_id=None,
workspace_id=None
):

results = []
with self.session() as session:
workbench_workspace_snapshot_alias = aliased(WorkbenchWorkspaceSnapshot)
active_id = session.query(func.max(WorkbenchWorkspaceSnapshot.id))\
Expand Down Expand Up @@ -355,119 +355,125 @@ def get_redcap_audit_workspaces(
if workspace_id:
query = query.filter(WorkbenchWorkspaceSnapshot.workspaceSourceId == workspace_id)\
.order_by(desc(subquery.c.id)).limit(1)
return self.get_redcap_audit_workspaces_in_list(query.all())
elif snapshot_id:
query = query.filter(subquery.c.id == snapshot_id)
elif last_snapshot_id:
query = query.filter(subquery.c.id > last_snapshot_id)\
return self.get_redcap_audit_workspaces_in_list(query.all())
elif self.last_snapshot_id:
query = query.filter(subquery.c.id > self.last_snapshot_id)\
.order_by(subquery.c.id)
items = query.all()

for workspace, researcher, current_id, current_status in items:
verified_institutional_affiliation = {}
affiliations = []
if researcher.workbenchInstitutionalAffiliations:
for affiliation in researcher.workbenchInstitutionalAffiliations:
affiliations.append(
{
"institution": affiliation.institution,
"role": affiliation.role,
"isVerified": affiliation.isVerified,
"nonAcademicAffiliation":
str(WorkbenchInstitutionNonAcademic(affiliation.nonAcademicAffiliation))
if affiliation.nonAcademicAffiliation else 'UNSET'
}
)
if affiliation.isVerified:
verified_institutional_affiliation = {
"institution": affiliation.institution,
"role": affiliation.role,
"nonAcademicAffiliation":
str(WorkbenchInstitutionNonAcademic(affiliation.nonAcademicAffiliation))
if affiliation.nonAcademicAffiliation else 'UNSET'
}
workspace_researcher = {
"userId": researcher.userSourceId,
"creationTime": researcher.creationTime,
"modifiedTime": researcher.modifiedTime,
"givenName": researcher.givenName,
"familyName": researcher.familyName,
"email": researcher.email,
"accessTier": researcher.get_access_tier(),
"verifiedInstitutionalAffiliation": verified_institutional_affiliation,
"affiliations": affiliations
}
return query

exist = False
for result in results:
if result['snapshotId'] == current_id:
result['workspaceResearchers'].append(workspace_researcher)
exist = True
break
if exist:
continue
record = {
'snapshotId': current_id,
'workspaceId': workspace.workspaceSourceId,
'name': workspace.name,
'creationTime': workspace.creationTime,
'modifiedTime': workspace.modifiedTime,
'status': str(WorkbenchWorkspaceStatus(current_status)),
'workspaceUsers': [
def get_redcap_audit_workspaces_in_list(self, items):
results = []

for workspace, researcher, current_id, current_status in items:
verified_institutional_affiliation = {}
affiliations = []
if researcher.workbenchInstitutionalAffiliations:
for affiliation in researcher.workbenchInstitutionalAffiliations:
affiliations.append(
{
"userId": user.userId,
"role": str(WorkbenchWorkspaceUserRole(user.role)) if user.role else 'UNSET',
"status": str(WorkbenchWorkspaceStatus(user.status)) if user.status else 'UNSET',
"isCreator": user.isCreator
} for user in workspace.workbenchWorkspaceUser
] if workspace.workbenchWorkspaceUser else [],
'workspaceResearchers': [workspace_researcher],
"excludeFromPublicDirectory": workspace.excludeFromPublicDirectory,
"ethicalLegalSocialImplications": workspace.ethicalLegalSocialImplications,
"reviewRequested": workspace.reviewRequested if workspace.reviewRequested else False,
"diseaseFocusedResearch": workspace.diseaseFocusedResearch,
"diseaseFocusedResearchName": workspace.diseaseFocusedResearchName,
"otherPurposeDetails": workspace.otherPurposeDetails,
"methodsDevelopment": workspace.methodsDevelopment,
"controlSet": workspace.controlSet,
"ancestry": workspace.ancestry,
"socialBehavioral": workspace.socialBehavioral,
"populationHealth": workspace.populationHealth,
"drugDevelopment": workspace.drugDevelopment,
"commercialPurpose": workspace.commercialPurpose,
"educational": workspace.educational,
"otherPurpose": workspace.otherPurpose,
"accessTier": str(WorkbenchWorkspaceAccessTier(workspace.accessTier
if workspace.accessTier else 0)),
"scientificApproaches": workspace.scientificApproaches,
"intendToStudy": workspace.intendToStudy,
"findingsFromStudy": workspace.findingsFromStudy,
"focusOnUnderrepresentedPopulations": workspace.focusOnUnderrepresentedPopulations,
"workspaceDemographic": {
"raceEthnicity": [str(WorkbenchWorkspaceRaceEthnicity(value))
for value in workspace.raceEthnicity] if workspace.raceEthnicity else None,
"age": [str(WorkbenchWorkspaceAge(value)) for value in workspace.age]
if workspace.age else None,
"sexAtBirth": str(WorkbenchWorkspaceSexAtBirth(workspace.sexAtBirth))
if workspace.sexAtBirth else None,
"genderIdentity": str(WorkbenchWorkspaceGenderIdentity(workspace.genderIdentity))
if workspace.genderIdentity else None,
"sexualOrientation": str(WorkbenchWorkspaceSexualOrientation(workspace.sexualOrientation))
if workspace.sexualOrientation else None,
"geography": str(WorkbenchWorkspaceGeography(workspace.geography))
if workspace.geography else None,
"disabilityStatus": str(WorkbenchWorkspaceDisabilityStatus(workspace.disabilityStatus))
if workspace.disabilityStatus else None,
"accessToCare": str(WorkbenchWorkspaceAccessToCare(workspace.accessToCare))
if workspace.accessToCare else None,
"educationLevel": str(WorkbenchWorkspaceEducationLevel(workspace.educationLevel))
if workspace.educationLevel else None,
"incomeLevel": str(WorkbenchWorkspaceIncomeLevel(workspace.incomeLevel))
if workspace.incomeLevel else None,
"others": workspace.others
},
"cdrVersion": workspace.cdrVersion
}
results.append(record)
"institution": affiliation.institution,
"role": affiliation.role,
"isVerified": affiliation.isVerified,
"nonAcademicAffiliation":
str(WorkbenchInstitutionNonAcademic(affiliation.nonAcademicAffiliation))
if affiliation.nonAcademicAffiliation else 'UNSET'
}
)
if affiliation.isVerified:
verified_institutional_affiliation = {
"institution": affiliation.institution,
"role": affiliation.role,
"nonAcademicAffiliation":
str(WorkbenchInstitutionNonAcademic(affiliation.nonAcademicAffiliation))
if affiliation.nonAcademicAffiliation else 'UNSET'
}
workspace_researcher = {
"userId": researcher.userSourceId,
"creationTime": researcher.creationTime,
"modifiedTime": researcher.modifiedTime,
"givenName": researcher.givenName,
"familyName": researcher.familyName,
"email": researcher.email,
"accessTier": researcher.get_access_tier(),
"verifiedInstitutionalAffiliation": verified_institutional_affiliation,
"affiliations": affiliations
}

exist = False
for result in results:
if result['snapshotId'] == current_id:
result['workspaceResearchers'].append(workspace_researcher)
exist = True
break
if exist:
continue
record = {
'snapshotId': current_id,
'workspaceId': workspace.workspaceSourceId,
'name': workspace.name,
'creationTime': workspace.creationTime,
'modifiedTime': workspace.modifiedTime,
'status': str(WorkbenchWorkspaceStatus(current_status)),
'workspaceUsers': [
{
"userId": user.userId,
"role": str(WorkbenchWorkspaceUserRole(user.role)) if user.role else 'UNSET',
"status": str(WorkbenchWorkspaceStatus(user.status)) if user.status else 'UNSET',
"isCreator": user.isCreator
} for user in workspace.workbenchWorkspaceUser
] if workspace.workbenchWorkspaceUser else [],
'workspaceResearchers': [workspace_researcher],
"excludeFromPublicDirectory": workspace.excludeFromPublicDirectory,
"ethicalLegalSocialImplications": workspace.ethicalLegalSocialImplications,
"reviewRequested": workspace.reviewRequested if workspace.reviewRequested else False,
"diseaseFocusedResearch": workspace.diseaseFocusedResearch,
"diseaseFocusedResearchName": workspace.diseaseFocusedResearchName,
"otherPurposeDetails": workspace.otherPurposeDetails,
"methodsDevelopment": workspace.methodsDevelopment,
"controlSet": workspace.controlSet,
"ancestry": workspace.ancestry,
"socialBehavioral": workspace.socialBehavioral,
"populationHealth": workspace.populationHealth,
"drugDevelopment": workspace.drugDevelopment,
"commercialPurpose": workspace.commercialPurpose,
"educational": workspace.educational,
"otherPurpose": workspace.otherPurpose,
"accessTier": str(WorkbenchWorkspaceAccessTier(workspace.accessTier
if workspace.accessTier else 0)),
"scientificApproaches": workspace.scientificApproaches,
"intendToStudy": workspace.intendToStudy,
"findingsFromStudy": workspace.findingsFromStudy,
"focusOnUnderrepresentedPopulations": workspace.focusOnUnderrepresentedPopulations,
"workspaceDemographic": {
"raceEthnicity": [str(WorkbenchWorkspaceRaceEthnicity(value))
for value in workspace.raceEthnicity] if workspace.raceEthnicity else None,
"age": [str(WorkbenchWorkspaceAge(value)) for value in workspace.age]
if workspace.age else None,
"sexAtBirth": str(WorkbenchWorkspaceSexAtBirth(workspace.sexAtBirth))
if workspace.sexAtBirth else None,
"genderIdentity": str(WorkbenchWorkspaceGenderIdentity(workspace.genderIdentity))
if workspace.genderIdentity else None,
"sexualOrientation": str(WorkbenchWorkspaceSexualOrientation(workspace.sexualOrientation))
if workspace.sexualOrientation else None,
"geography": str(WorkbenchWorkspaceGeography(workspace.geography))
if workspace.geography else None,
"disabilityStatus": str(WorkbenchWorkspaceDisabilityStatus(workspace.disabilityStatus))
if workspace.disabilityStatus else None,
"accessToCare": str(WorkbenchWorkspaceAccessToCare(workspace.accessToCare))
if workspace.accessToCare else None,
"educationLevel": str(WorkbenchWorkspaceEducationLevel(workspace.educationLevel))
if workspace.educationLevel else None,
"incomeLevel": str(WorkbenchWorkspaceIncomeLevel(workspace.incomeLevel))
if workspace.incomeLevel else None,
"others": workspace.others
},
"cdrVersion": workspace.cdrVersion
}
results.append(record)

return results

Expand Down Expand Up @@ -1304,7 +1310,7 @@ def get_all_with_children(self):

class WorkbenchWorkspaceAuditDao(UpdatableDao):
def __init__(self):
super().__init__(WorkbenchWorkspaceAuditDao, order_by_ending=["id"])
super().__init__(WorkbenchWorkspaceSnapshot, order_by_ending=["id"])
self.workspace_dao = WorkbenchWorkspaceDao()
self.workspace_snapshot_dao = WorkbenchWorkspaceHistoryDao()

Expand Down Expand Up @@ -1376,6 +1382,46 @@ def to_client_json(self, obj):
for record in obj:
result.append(json.loads(record.resource))
return result
elif isinstance(obj, dict):
return [obj]

def _initialize_query(self, session, query_def):
return self.workspace_dao.get_redcap_audit_workspaces()

def query(self, query_definition):
if query_definition.invalid_filters and not query_definition.field_filters:
raise BadRequest("No valid fields were provided")
if not self.order_by_ending:
raise BadRequest(f"Can't query on type {self.model_type} -- no order by ending specified")
with self.session() as session:
total = None
query, field_names = self._make_query(session, query_definition)

# For WorkbenchWorkspaceAudit, we want to set the results limit after the results have been placed in a list
workbench_max_results = query_definition.max_results + 1
query_results = query.with_session(session).limit(None).all()
items = self.workspace_dao.get_redcap_audit_workspaces_in_list(query_results)[:workbench_max_results]

if query_definition.include_total:
total = self._count_query(session, query_definition)
if not items:
return Results([], total=total)

if len(items) > query_definition.max_results:
# Items, pagination token, and more are available
page = items[0: query_definition.max_results]
token = self._make_pagination_token(
items[query_definition.max_results - 1],
field_names=['snapshotId']
)
return Results(page, token, more_available=True, total=total)
else:
token = (
self._make_pagination_token(
items[-1].asdict(),
field_names) if query_definition.always_return_token else None
)
return Results(items, token, more_available=False, total=total)

def remove_approved_workspace_with_session(self, session, workspace_snapshot_id):
workspace_snapshot = self.workspace_snapshot_dao.get_snapshot_by_id_with_session(session, workspace_snapshot_id)
Expand Down
Loading