diff --git a/src/renderer/src/context/Projects/index.tsx b/src/renderer/src/context/Projects/index.tsx index a6c6c30..b02a239 100644 --- a/src/renderer/src/context/Projects/index.tsx +++ b/src/renderer/src/context/Projects/index.tsx @@ -177,7 +177,6 @@ export const ProjectsProvider: React.FC<{ children: ReactNode }> = ({ }, []) const createMotifVideos = useCallback(async (data) => { - console.log(data) const res = await createMotifVideosVAMEProject(data) await refresh() return res diff --git a/src/renderer/src/pages/Project/index.tsx b/src/renderer/src/pages/Project/index.tsx index 840c3b7..9ffd194 100644 --- a/src/renderer/src/pages/Project/index.tsx +++ b/src/renderer/src/pages/Project/index.tsx @@ -57,7 +57,6 @@ const Project: React.FC = () => { if (tab) { const localItem = `selected-tab-${project?.config.Project}` - console.log("localItem",localItem) localStorage.setItem(localItem, tab) setSelectedTab(tab) } @@ -109,6 +108,7 @@ const Project: React.FC = () => { segmented, motif_videos_created, community_videos_created, + umaps_created, } = project.workflow const { @@ -261,7 +261,7 @@ const Project: React.FC = () => { { id: 'umap-visualization', label: '7. UMAP Visualization', - completed: visualization?.execution_state==="success", + complete: umaps_created, disabled: !segmented, tooltip: "Need segmentation.", content: str: - with open(csv_file_path) as csv_file: - csv_reader = csv.reader(csv_file, delimiter=',') - body_parts = [] - - # loop to iterate through the rows of csv - for row in csv_reader: - if row[0] == "bodyparts": - body_parts = list(dict.fromkeys(row[1:])) # Extract body parts starting from the second element and remove duplicates - break - - if len(body_parts) == 0: - print("No body parts headers found in CSV.") - return "" - - # Create the string based on body parts - body_parts_string = ", ".join([f"{i}-{part}" for i, part in enumerate(body_parts)]) - - return body_parts_string - - -@api.route('/connected') -class Connected(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def get(self): - return jsonify({ "payload": True }) - - -@api.route('/ready') -class VAMEReady(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def get(self): - import vame - return jsonify({ "payload": True }) - -@api.route('/project_ready') -class ProjectReady(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - data, project_path = resolve_request_data(request) - - states_path = Path(project_path) / "states" / "states.json" - - with open(states_path, "r") as file: - states = json.load(fp=file) - - if states is None: - return jsonify({ "is_ready": True }) - - for _, value in states.items(): - execution_state = value.get("execution_state", None) - if execution_state == "running": - return jsonify({ "is_ready": False }) - - return jsonify({ "is_ready": True }) - -@api.route('/settings') -class Settings(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def get(self): - return json.loads(open(GLOBAL_SETTINGS_FILE, "r").read()) - - - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - data = json.loads(request.data) if request.data else {} - with open(GLOBAL_SETTINGS_FILE, "w") as file: - json.dump(data, file) - return jsonify(data) - -@api.route('/files//') -class Files(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def get(self, project, path): - from flask import send_from_directory - return send_from_directory(VAME_PROJECTS_DIRECTORY, Path(project) / path) - - -@api.route('/exists//') -class FileExists(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def get(self, project, path): - full_path = VAME_PROJECTS_DIRECTORY / project / path - return jsonify(dict(exists=full_path.exists())) - - -@api.route('/projects') -class Projects(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def get(self): - projects = [ str(project) for project in VAME_PROJECTS_DIRECTORY.glob('*') if project.is_dir() ] - return jsonify(projects) - -@api.route('/projects/recent') -class RecentProjects(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def get(self): - with open(GLOBAL_STATES_FILE, "r") as inp: - states = json.load(fp=inp) - # states = json.loads(open(GLOBAL_STATES_FILE, "r").read()) - recent_projects = states.get("recent_projects", []) - - # Filter those that no longer exist - recent_projects = [ str(project) for project in recent_projects if (VAME_PROJECTS_DIRECTORY / project).exists() ] - states["recent_projects"] = recent_projects - with open(GLOBAL_STATES_FILE, "w") as file: - json.dump(states, file) - - return jsonify(recent_projects) - -@api.route('/project/register') -class RegisterProject(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - - try: - with open(GLOBAL_STATES_FILE, "r") as inp: - states = json.load(fp=inp) - - recent_projects = states.get("recent_projects", []) - - _, project_path = resolve_request_data(request) - - project_path = str(project_path) - - if project_path in recent_projects: - recent_projects.remove(project_path) - - recent_projects.append(project_path) - - if len(recent_projects) > 5: - recent_projects = recent_projects[-5:] - - with open(GLOBAL_STATES_FILE, "w") as file: - json.dump({ - **states, - "recent_projects": recent_projects - }, file) - - return jsonify(recent_projects) - - except Exception as exception: - # TODO: Should lock access to the file - pass - -@api.route('/log/') -class ProjectLog(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request",404: "Not found", 500: "Internal server error"}) - def get(self, log_name): - - project_path = request.args.get('project') - project_path = Path(project_path) - - if(log_name): - log_path = Path(project_path) / "logs" / f"{log_name}.log" - if log_path.is_file(): - log = open(log_path, "r").read() - return log - else: - return f"{log_name} log not found.", 404 - - return "missing log_name", 400 - -@api.route('/load') -class Load(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - - _, project_path = resolve_request_data(request) - config_path = project_path / "config.yaml" - - # Create a symlink to the project directory if it isn't in the VAME_PROJECTS_DIRECTORY - if project_path.parent != VAME_PROJECTS_DIRECTORY: - symlink = VAME_PROJECTS_DIRECTORY / project_path.name - if not symlink.exists(): - symlink.symlink_to(project_path) - - states_path = Path(project_path) / "states" / "states.json" - - states = json.load(open(states_path)) if os.path.exists(states_path) else None - - config = yaml.safe_load(open(config_path, "r")) if config_path.exists() else None - - images = dict( - evaluation=get_evaluation_images(project_path), - visualization=get_visualization_images(project_path) - ) - - videos = dict( - motif=get_motif_videos(project_path), - community=get_community_videos(project_path) - ) - - has_latent_vector_files = False - if config: - has_latent_vector_files = all(map(lambda video: (get_video_results_path(video, project_path) / f"latent_vector_{video}.npy").exists(), config["video_sets"])) - - has_communities = (project_path / 'cohort_community_label.npy').exists() - - original_videos_location = project_path / 'videos' - original_csvs_location = original_videos_location / 'pose_estimation' - - # Get all files in the original data directory - original_videos = list(map(lambda file: str(file), get_files(original_videos_location))) - original_csvs = list(map(lambda file: str(file), get_files(original_csvs_location))) - - # Provide project workflow status - workflow = dict( - organized = (project_path / 'data' / 'train').exists(), - pose_ref_index_description=get_pose_ref_index_description(original_csvs[0]), - modeled = len(images["evaluation"]) > 0, - segmented = has_latent_vector_files, - motif_videos_created = all(map(lambda videos: len(videos) > 0, videos["motif"].values())), - communities_created = has_communities, - community_videos_created = all(map(lambda videos: len(videos) > 0, videos["community"].values())), - umaps_created = any(map(lambda videos: len(videos) > 0, images["visualization"].values())), - ) - - return jsonify(dict( - project=str(config_path.parent), - config=config, - assets=dict( - images=images, - videos=videos - ), - videos=original_videos, - csvs=original_csvs, - workflow=workflow, - states=states - )) - -@api.route('/create', methods=['POST']) -class Create(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - import vame - try: - - data = json.loads(request.data) if request.data else {} - - project_path = get_project_path(data["project"], VAME_PROJECTS_DIRECTORY) - - created = not project_path.exists() - - config_path = Path(vame.init_new_project( - working_directory=VAME_PROJECTS_DIRECTORY, - **data - )) - - return jsonify(dict( - project=str(config_path.parent), - created=created, - config=yaml.safe_load(open(config_path, "r")) - )) - - except Exception as exception: - print("exception", exception) - if notBadRequestException(exception): - api.abort(500, str(exception)) - -@api.route('/delete_project', methods=['POST']) -class DeleteProject(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - try: - data, project_path = resolve_request_data(request) - - if project_path.exists(): - import shutil - shutil.rmtree(project_path, ignore_errors=True) - return jsonify(dict(project=str(project_path), deleted=True)) - - return jsonify(dict(project=str(project_path), deleted=False)) - - except Exception as exception: - if notBadRequestException(exception): - api.abort(500, str(exception)) - - -@api.route('/configure', methods=['POST']) -class ConfigureProject(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - from vame.util import auxiliary - - try: - data, project_path = resolve_request_data(request) - config_path = project_path / "config.yaml" - - if config_path.exists(): - with open(config_path, "r") as file: - config = yaml.safe_load(file) - config_update = data["config"] - - if config_update: - config.update(config_update) - auxiliary.write_config(config_path, config) - - return jsonify(dict(config=config)) - - return jsonify(dict(config=None)) - - except Exception as exception: - if notBadRequestException(exception): - api.abort(500, str(exception)) - - -@api.route('/align', methods=['POST']) -class Align(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - import vame - try: - - data, project_path = resolve_request_data(request) - - # If your experiment is by design egocentrical (e.g. head-fixed experiment on treadmill etc) - # you can use the following to convert your .csv to a .npy array, ready to train vame on it - egocentric_data = data.pop('egocentric_data', None) - - - if egocentric_data: - vame.csv_to_numpy( - project_path / 'config.yaml', - save_logs=True - ) - - else: - vame.egocentric_alignment( - **data, - save_logs=True - ) - - - return jsonify(dict(result='success')) - - except Exception as exception: - if notBadRequestException(exception): - api.abort(500, str(exception)) - - -@api.route("/create_trainset") -class CreateTrainset(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - import vame - try: - data, project_path = resolve_request_data(request) - - result = vame.create_trainset( - **data, - save_logs=True - ) - - return dict(result=result) - except Exception as exception: - if notBadRequestException(exception): - api.abort(500, str(exception)) - -@api.route('/train', methods=['POST']) -class TrainModel(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - import vame - try: - data, project_path = resolve_request_data(request) - - result = vame.train_model( - **data, - save_logs=True - ) - - return dict(result=result) - except Exception as exception: - if notBadRequestException(exception): - api.abort(500, str(exception)) - -@api.route('/evaluate', methods=['POST']) -class EvaluateModel(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - import vame - try: - data, project_path = resolve_request_data(request) - vame.evaluate_model( - **data, - save_logs=True - ) - return dict(result=get_evaluation_images(project_path)) - except Exception as exception: - if notBadRequestException(exception): - api.abort(500, str(exception)) - -@api.route('/segment', methods=['POST']) -class Segment(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - import vame - try: - data, project_path = resolve_request_data(request) - result = vame.pose_segmentation( - **data, - save_logs=True - ) - return dict(result=result) - except Exception as exception: - if notBadRequestException(exception): - api.abort(500, str(exception)) - -@api.route('/motif_videos', methods=['POST']) -class MotifVideos(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - import vame - try: - data, project_path = resolve_request_data(request) - result = vame.motif_videos( - **data, - save_logs=True - ) - return dict(result=result) - except Exception as exception: - if notBadRequestException(exception): - api.abort(500, str(exception)) - -@api.route('/community', methods=['POST']) -class Community(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - import vame - try: - data, project_path = resolve_request_data(request) - result = vame.community( - **data, - save_logs=True - ) - return dict(result=result) - except Exception as exception: - if notBadRequestException(exception): - api.abort(500, str(exception)) - -@api.route('/community_videos', methods=['POST']) -class CommunityVideos(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - import vame - try: - data, project_path = resolve_request_data(request) - result = vame.community_videos( - **data, - save_logs=True - ) - return dict(result=result) - except Exception as exception: - if notBadRequestException(exception): - api.abort(500, str(exception)) - -@api.route('/visualization', methods=['POST']) -class Visualization(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - import vame - try: - data, project_path = resolve_request_data(request) - vame.visualization( - **data, - save_logs=True - ) - return dict(result=get_visualization_images(project_path)) - except Exception as exception: - if notBadRequestException(exception): - api.abort(500, str(exception)) - -@api.route('/generative_model', methods=['POST']) -class GenerativeModel(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - import vame - try: - data, project_path = resolve_request_data(request) - result = vame.generative_model(**data) - return dict(result=result) - except Exception as exception: - if notBadRequestException(exception): - api.abort(500, str(exception)) - -@api.route('/gif', methods=['POST']) -class CreateGif(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - import vame - try: - data, project_path = resolve_request_data(request) - result = vame.gif(**data) - return dict(result=result) - except Exception as exception: - if notBadRequestException(exception): - api.abort(500, str(exception)) - -def signal_handler(sig, frame): - print('Received SIGTERM, shutting down...') - os._exit(0) - -signal.signal(signal.SIGTERM, signal_handler) - -if __name__ == "__main__": - env_port = os.getenv('PORT') - PORT = int(env_port) if env_port else 8641 - HOST = os.getenv('HOST') or 'localhost' - - VAME_PROJECTS_DIRECTORY.mkdir(exist_ok=True, parents=True) # Create the VAME_PROJECTS_DIRECTORY if it doesn't exist - VAME_LOG_DIRECTORY.mkdir(exist_ok=True, parents=True) # Create the VAME_LOG_DIRECTORY if it doesn't exist - - # Create the global files if they don't exist - global_files = [GLOBAL_STATES_FILE, GLOBAL_SETTINGS_FILE] - for file in global_files: - if not file.exists(): - with open(file, "w") as f: - json.dump({}, f) - - # Run the app - try: - print(f"Flask server started at {HOST}:{PORT}") - app.run(host=HOST, port = PORT) - - except Exception as e: - print(f"An error occurred that closed the server: {e}") - raise e diff --git a/src/services/vameApi/app/routes/vame.py b/src/services/vameApi/app/routes/vame.py index 81a8aef..8f9c940 100644 --- a/src/services/vameApi/app/routes/vame.py +++ b/src/services/vameApi/app/routes/vame.py @@ -11,6 +11,8 @@ class Align(Resource): @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): import vame + import matplotlib + matplotlib.use('agg') try: data, project_path = resolve_request_data(request) @@ -45,6 +47,8 @@ class CreateTrainset(Resource): @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): import vame + import matplotlib + matplotlib.use('agg') try: data, project_path = resolve_request_data(request) @@ -63,6 +67,8 @@ class TrainModel(Resource): @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): import vame + import matplotlib + matplotlib.use('agg') try: data, project_path = resolve_request_data(request) @@ -99,6 +105,8 @@ class Segment(Resource): @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): import vame + import matplotlib + matplotlib.use('agg') try: data, project_path = resolve_request_data(request) result = vame.pose_segmentation( @@ -115,6 +123,8 @@ class MotifVideos(Resource): @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): import vame + import matplotlib + matplotlib.use('agg') try: data, project_path = resolve_request_data(request) result = vame.motif_videos( @@ -131,6 +141,8 @@ class Community(Resource): @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): import vame + import matplotlib + matplotlib.use('agg') try: data, project_path = resolve_request_data(request) result = vame.community( @@ -147,6 +159,8 @@ class CommunityVideos(Resource): @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): import vame + import matplotlib + matplotlib.use('agg') try: data, project_path = resolve_request_data(request) result = vame.community_videos( @@ -163,6 +177,8 @@ class Visualization(Resource): @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): import vame + import matplotlib + matplotlib.use('agg') try: data, project_path = resolve_request_data(request) vame.visualization( @@ -179,6 +195,8 @@ class GenerativeModel(Resource): @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): import vame + import matplotlib + matplotlib.use('agg') try: data, project_path = resolve_request_data(request) result = vame.generative_model(**data) @@ -192,6 +210,8 @@ class CreateGif(Resource): @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): import vame + import matplotlib + matplotlib.use('agg') try: data, project_path = resolve_request_data(request) result = vame.gif(**data) diff --git a/src/services/vameApi/app/services/project_service.py b/src/services/vameApi/app/services/project_service.py index 0b4e538..483f605 100644 --- a/src/services/vameApi/app/services/project_service.py +++ b/src/services/vameApi/app/services/project_service.py @@ -157,17 +157,26 @@ def load_project(project_path: Path): original_videos = list(map(lambda file: str(file), get_files(original_videos_location))) original_csvs = list(map(lambda file: str(file), get_files(original_csvs_location))) + motif_videos_created_hmm=all(map(lambda videos: len(videos) > 0, videos["motif"]["hmm"].values())) + motif_videos_created_kmeans=all(map(lambda videos: len(videos) > 0, videos["motif"]["kmeans"].values())) + + community_videos_created_hmm = all(map(lambda videos: len(videos) > 0, videos["community"]["kmeans"].values())) + community_videos_created_kmeans = all(map(lambda videos: len(videos) > 0, videos["community"]["hmm"].values())) + + umaps_created_hmm = any(map(lambda videos: len(videos) > 0, images["visualization"]["hmm"].values())) + + umaps_created_kmeans = any(map(lambda videos: len(videos) > 0, images["visualization"]["kmeans"].values())) + # Provide project workflow status workflow = dict( organized = (project_path / 'data' / 'train').exists(), pose_ref_index_description=get_pose_ref_index_description(original_csvs[0]), modeled = len(images["evaluation"]) > 0, segmented = has_latent_vector_files, - motif_videos_created_hmm = all(map(lambda videos: len(videos) > 0, videos["motif"]["hmm"].values())), - motif_videos_created_kmeans = all(map(lambda videos: len(videos) > 0, videos["motif"]["kmeans"].values())), - communities_created = has_communities, - community_videos_created = all(map(lambda videos: len(videos) > 0, videos["community"].values())), - umaps_created = any(map(lambda videos: len(videos) > 0, images["visualization"].values())), + motif_videos_created=(lambda: motif_videos_created_hmm or motif_videos_created_kmeans)(), + communities_created=has_communities, + community_videos_created = (lambda: community_videos_created_hmm or community_videos_created_kmeans)(), + umaps_created = (lambda: umaps_created_hmm or umaps_created_kmeans)(), ) return dict(