From 16ac54bf4e416938337058b6e02256162cdfef34 Mon Sep 17 00:00:00 2001 From: realdragonturtle <29815169+realdragonturtle@users.noreply.github.com> Date: Tue, 1 Feb 2022 21:22:35 +0100 Subject: [PATCH 01/20] added additional gui window --- .gitignore | 1 + bin/mediapipepose.py | 303 +++++++++++++++++++++++++++++++------------ 2 files changed, 219 insertions(+), 85 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..474c8f2 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +env\* \ No newline at end of file diff --git a/bin/mediapipepose.py b/bin/mediapipepose.py index e02d069..b7673e6 100644 --- a/bin/mediapipepose.py +++ b/bin/mediapipepose.py @@ -1,10 +1,15 @@ print("Importing libraries....") +from math import degrees import os import sys +from turtle import width sys.path.append(os.getcwd()) #embedable python doesnt find local modules without this line +import tkinter as tk +from tkinter import ttk + import time import threading import cv2 @@ -17,6 +22,8 @@ mp_drawing = mp.solutions.drawing_utils mp_pose = mp.solutions.pose +use_steamvr = False + print("Reading parameters...") param = getparams() @@ -39,12 +46,14 @@ use_hands = param["use_hands"] ignore_hip = param["ignore_hip"] +prev_smoothing = smoothing + print("Opening camera...") image_from_thread = None image_ready = False -def camera_thread(): +def camera_thread_fun(): #A seperate camera thread that captures images and saves it to a variable to be read by the main program. #Mostly used to free the main thread from image decoding overhead and to ensure frames are just skipped if detection is slower than camera fps global cameraid, image_from_thread, image_ready @@ -61,8 +70,134 @@ def camera_thread(): assert ret, "Camera capture failed! Check the cameraid parameter." -thread = threading.Thread(target=camera_thread, daemon=True) -thread.start() #start our thread, which starts camera capture +recalibrate = False +def change_recalibrate(): + global recalibrate + recalibrate = True + +global_rot_y = R.from_euler('y',0,degrees=True) #default rotations, for 0 degrees around y and x +global_rot_x = R.from_euler('x',0,degrees=True) +global_rot_z = R.from_euler('z',0,degrees=True) +def rot_change_y(value): + global global_rot_y #callback functions. Whenever the value on sliders are changed, they are called + global_rot_y = R.from_euler('y',value,degrees=True) #and the rotation is updated with the new value. + +def rot_change_x(value): + global global_rot_x + global_rot_x = R.from_euler('x',value-90,degrees=True) + +def rot_change_z(value): + global global_rot_z + global_rot_z = R.from_euler('z',value-180,degrees=True) + +posescale = 1 +def change_scale(value): + global posescale + posescale = value/50 + 0.5 + +camera_thread = threading.Thread(target=camera_thread_fun, daemon=True) +camera_thread.start() #start our thread, which starts camera capture + +exit_ready = False + +def gui_thread_fun(): + global smoothing, calib_rot, calib_tilt, change_recalibrate, rot_change_x, rot_change_y, rot_change_z, global_rot_x, global_rot_y, global_rot_z + + def ready2exit(): + global exit_ready + exit_ready = True + + def update_vals(smoothing_val): + global smoothing + smoothing = smoothing_val + + window = tk.Tk() + + frame1 = tk.Frame(window) + frame1.pack() + + def show_hide(value, frames): + val = value.get() + for frame in frames: + if not val: + frame.pack() #.grid(row=1) + else: + frame.pack_forget() # .grid_forget() + + ### calib rot + + varrot = tk.IntVar(value = calib_rot) + rot_check = tk.Checkbutton(frame1, text = "Enable automatic rotation calibration", variable = varrot, command=lambda *args: show_hide(varrot, [rot_y_frame])) + rot_check.pack() + calib_rot = bool(varrot.get()) + + rot_y_frame = tk.Frame(frame1) + rot_y_frame.pack() + + rot_y = tk.Scale(rot_y_frame, label="Roation y:", from_=0, to=360, command=rot_change_y, orient=tk.HORIZONTAL, + length=800, showvalue=1, tickinterval=50) + rot_y.pack(expand=True,fill='both',side='left') + rot_y.set(global_rot_y.as_euler('zyx', degrees=True)[1]) + + tk.Button(rot_y_frame, text="<", command= lambda *args: rot_y.set(rot_y.get()-1), width=10).pack(expand=True,fill='both',side='left') + tk.Button(rot_y_frame, text=">", command= lambda *args: rot_y.set(rot_y.get()+1), width=10).pack(expand=True,fill='both',side='left') + + separator = ttk.Separator(window, orient='horizontal') + separator.pack(fill='x') + + ## calib tilt + + frame2 = tk.Frame(window) + frame2.pack() + + vartilt = tk.IntVar(value = calib_tilt) + tilt_check = tk.Checkbutton(frame2, text = "Enable automatic tilt calibration", variable = vartilt, command=lambda *args: show_hide(vartilt, [rot_z_frame, rot_x_frame])) + tilt_check.pack() + calib_tilt = bool(vartilt.get()) + + rot_x_frame = tk.Frame(frame2) + rot_x_frame.pack() + rot_z_frame = tk.Frame(frame2) + rot_z_frame.pack() + + rot_x = tk.Scale(rot_x_frame, label="Roation x:", from_=90, to=180, command=rot_change_x, orient=tk.HORIZONTAL, + length=800, showvalue=1, tickinterval=30) + rot_x.pack(expand=True,fill='both',side='left') + rot_x.set(global_rot_x.as_euler('zyx', degrees=True)[2]) + tk.Button(rot_x_frame, text="<", command= lambda *args: rot_x.set(rot_x.get()-1), width=10).pack(expand=True,fill='both',side='left') + tk.Button(rot_x_frame, text=">", command= lambda *args: rot_x.set(rot_x.get()+1), width=10).pack(expand=True,fill='both',side='left') + + rot_z = tk.Scale(rot_z_frame, label="Roation z:", from_=180, to=360, command=rot_change_z, orient=tk.HORIZONTAL, + length=800, showvalue=1, tickinterval=30) + rot_z.pack(expand=True,fill='both',side='left') + rot_z.set(global_rot_z.as_euler('zyx', degrees=True)[0 ]) + tk.Button(rot_z_frame, text="<", command= lambda *args: rot_z.set(rot_z.get()-1), width=10).pack(expand=True,fill='both',side='left') + tk.Button(rot_z_frame, text=">", command= lambda *args: rot_z.set(rot_z.get()+1), width=10).pack(expand=True,fill='both',side='left') + + separator = ttk.Separator(window, orient='horizontal') + separator.pack(fill='x') + + + bottom_frame = tk.Frame(window) + bottom_frame.pack() + + ## recalibrate + + tk.Button(bottom_frame, text='Recalibrate', command=change_recalibrate).pack() + + tk.Label(bottom_frame, text="Smoothing:", width = 50).pack() + smoothingtext = tk.Entry(width = 50) + smoothingtext.pack() + smoothingtext.insert(0, smoothing) + + tk.Button(bottom_frame, text='Update', command=lambda *args: update_vals(float(smoothingtext.get()))).pack() + + tk.Button(bottom_frame, text='Press to exit', command=ready2exit).pack() + + window.mainloop() + +gui_thread = threading.Thread(target=gui_thread_fun, daemon=True) +gui_thread.start() def sendToSteamVR(text): #Function to send a string to my steamvr driver through a named pipe. @@ -82,25 +217,26 @@ def sendToSteamVR(text): return array -print("Connecting to SteamVR") - -#ask the driver, how many devices are connected to ensure we dont add additional trackers -#in case we restart the program -numtrackers = sendToSteamVR("numtrackers") -for i in range(10): +if use_steamvr: + print("Connecting to SteamVR") + + #ask the driver, how many devices are connected to ensure we dont add additional trackers + #in case we restart the program + numtrackers = sendToSteamVR("numtrackers") + for i in range(10): + if "error" in numtrackers: + print("Error in SteamVR connection. Retrying...") + time.sleep(1) + numtrackers = sendToSteamVR("numtrackers") + else: + break + if "error" in numtrackers: - print("Error in SteamVR connection. Retrying...") - time.sleep(1) - numtrackers = sendToSteamVR("numtrackers") - else: - break - -if "error" in numtrackers: - print("Could not connect to SteamVR after 10 retries!") - time.sleep(10) - assert 0, "Could not connect to SteamVR after 10 retries" + print("Could not connect to SteamVR after 10 retries!") + time.sleep(10) + assert 0, "Could not connect to SteamVR after 10 retries" -numtrackers = int(numtrackers[2]) + numtrackers = int(numtrackers[2]) #games use 3 trackers, but we can also send the entire skeleton if we want to look at how it works totaltrackers = 23 if preview_skeleton else 3 @@ -121,20 +257,21 @@ def sendToSteamVR(text): for i in range(len(roles),totaltrackers): roles.append("None") -for i in range(numtrackers,totaltrackers): - #sending addtracker to our driver will... add a tracker. to our driver. - resp = sendToSteamVR(f"addtracker MediaPipeTracker{i} {roles[i]}") - while "error" in resp: +if use_steamvr: + for i in range(numtrackers,totaltrackers): + #sending addtracker to our driver will... add a tracker. to our driver. resp = sendToSteamVR(f"addtracker MediaPipeTracker{i} {roles[i]}") - print(resp) + while "error" in resp: + resp = sendToSteamVR(f"addtracker MediaPipeTracker{i} {roles[i]}") + print(resp) + time.sleep(0.2) time.sleep(0.2) - time.sleep(0.2) -resp = sendToSteamVR(f"settings 50 {smoothing}") -while "error" in resp: resp = sendToSteamVR(f"settings 50 {smoothing}") - print(resp) - time.sleep(1) + while "error" in resp: + resp = sendToSteamVR(f"settings 50 {smoothing}") + print(resp) + time.sleep(1) print("Starting pose detector...") @@ -144,33 +281,7 @@ def sendToSteamVR(text): min_tracking_confidence=0.5, model_complexity=1) -global_rot_y = R.from_euler('y',0,degrees=True) #default rotations, for 0 degrees around y and x -global_rot_x = R.from_euler('x',0,degrees=True) -global_rot_z = R.from_euler('z',0,degrees=True) -def rot_change_y(value): - global global_rot_y #callback functions. Whenever the value on sliders are changed, they are called - global_rot_y = R.from_euler('y',value,degrees=True) #and the rotation is updated with the new value. - -def rot_change_x(value): - global global_rot_x - global_rot_x = R.from_euler('x',value-90,degrees=True) - -def rot_change_z(value): - global global_rot_z - global_rot_z = R.from_euler('z',value-180,degrees=True) - -recalibrate = False -def change_recalibrate(value): - global recalibrate - if value == 1: - recalibrate = True - else: - recalibrate = False - -posescale = 1 -def change_scale(value): - global posescale - posescale = value/50 + 0.5 + cv2.namedWindow("out") if not calib_rot: @@ -190,6 +301,21 @@ def change_scale(value): i = 0 while(True): # Capture frame-by-frame + if exit_ready: + cv2.destroyAllWindows() + print("Exiting... You can close the window after 10 seconds.") + exit(0) + + if prev_smoothing != smoothing: + print(f"Changed smoothing value from {prev_smoothing} to {smoothing}") + prev_smoothing = smoothing + + if use_steamvr: + resp = sendToSteamVR(f"settings 50 {smoothing}") + while "error" in resp: + resp = sendToSteamVR(f"settings 50 {smoothing}") + print(resp) + time.sleep(1) if not image_ready: #wait untill camera thread captures another image time.sleep(0.001) @@ -238,9 +364,12 @@ def change_scale(value): if use_hands: hand_rots = get_rot_hands(pose3d) - array = sendToSteamVR("getdevicepose 0") #get hmd data to allign our skeleton to + if use_steamvr: + array = sendToSteamVR("getdevicepose 0") #get hmd data to allign our skeleton to + + if "error" in array: #continue to next iteration if there is an error + continue - if "error" not in array: headsetpos = [float(array[3]),float(array[4]),float(array[5])] headsetrot = R.from_quat([float(array[7]),float(array[8]),float(array[9]),float(array[6])]) @@ -268,9 +397,9 @@ def change_scale(value): value = np.arctan2(feet_middle[0],-feet_middle[1]) print("Postcalib z angle: ", value * 57.295779513) - + ##tilt calibration - + value = np.arctan2(feet_middle[2],-feet_middle[1]) print("Precalib x angle: ", value * 57.295779513) @@ -312,32 +441,36 @@ def change_scale(value): recalibrate = False else: - pose3d = pose3d * posescale #rescale skeleton to calibrated height - #print(pose3d) - offset = pose3d[7] - (headsetpos+neckoffset) #calculate the position of the skeleton - if not preview_skeleton: - numadded = 3 - if not ignore_hip: - for i in [(0,1),(5,2),(6,0)]: - joint = pose3d[i[0]] - offset #for each foot and hips, offset it by skeleton position and send to steamvr - sendToSteamVR(f"updatepose {i[1]} {joint[0]} {joint[1]} {joint[2]} {rots[i[1]][3]} {rots[i[1]][0]} {rots[i[1]][1]} {rots[i[1]][2]} {camera_latency} 0.8") + pose3d = pose3d * posescale #rescale skeleton to calibrated height + #print(pose3d) + offset = pose3d[7] - (headsetpos+neckoffset) #calculate the position of the skeleton + if not preview_skeleton: + numadded = 3 + if not ignore_hip: + for i in [(0,1),(5,2),(6,0)]: + joint = pose3d[i[0]] - offset #for each foot and hips, offset it by skeleton position and send to steamvr + if use_steamvr: + sendToSteamVR(f"updatepose {i[1]} {joint[0]} {joint[1]} {joint[2]} {rots[i[1]][3]} {rots[i[1]][0]} {rots[i[1]][1]} {rots[i[1]][2]} {camera_latency} 0.8") + else: + for i in [(0,0),(5,1)]: + joint = pose3d[i[0]] - offset #for each foot and hips, offset it by skeleton position and send to steamvr + if use_steamvr: + sendToSteamVR(f"updatepose {i[1]} {joint[0]} {joint[1]} {joint[2]} {rots[i[1]][3]} {rots[i[1]][0]} {rots[i[1]][1]} {rots[i[1]][2]} {camera_latency} 0.8") + numadded = 2 + if use_hands: + for i in [(10,0),(15,1)]: + joint = pose3d[i[0]] - offset #for each foot and hips, offset it by skeleton position and send to steamvr + if use_steamvr: + sendToSteamVR(f"updatepose {i[1]+numadded} {joint[0]} {joint[1]} {joint[2]} {hand_rots[i[1]][3]} {hand_rots[i[1]][0]} {hand_rots[i[1]][1]} {hand_rots[i[1]][2]} {camera_latency} 0.8") + + else: - for i in [(0,0),(5,1)]: - joint = pose3d[i[0]] - offset #for each foot and hips, offset it by skeleton position and send to steamvr - sendToSteamVR(f"updatepose {i[1]} {joint[0]} {joint[1]} {joint[2]} {rots[i[1]][3]} {rots[i[1]][0]} {rots[i[1]][1]} {rots[i[1]][2]} {camera_latency} 0.8") - numadded = 2 - if use_hands: - for i in [(10,0),(15,1)]: - joint = pose3d[i[0]] - offset #for each foot and hips, offset it by skeleton position and send to steamvr - sendToSteamVR(f"updatepose {i[1]+numadded} {joint[0]} {joint[1]} {joint[2]} {hand_rots[i[1]][3]} {hand_rots[i[1]][0]} {hand_rots[i[1]][1]} {hand_rots[i[1]][2]} {camera_latency} 0.8") - - - else: - for i in range(23): - joint = pose3d[i] - offset #if previewing skeleton, send the position of each keypoint to steamvr without rotation - sendToSteamVR(f"updatepose {i} {joint[0]} {joint[1]} {joint[2] - 2} 1 0 0 0 {camera_latency} 0.8") + for i in range(23): + joint = pose3d[i] - offset #if previewing skeleton, send the position of each keypoint to steamvr without rotation + if use_steamvr: + sendToSteamVR(f"updatepose {i} {joint[0]} {joint[1]} {joint[2] - 2} 1 0 0 0 {camera_latency} 0.8") - print("inference time:", time.time()-t0) #print how long it took to detect and calculate everything + print(f"Inference time: {time.time()-t0}\nSmoothing value: {smoothing}\n") #print how long it took to detect and calculate everything #cv2.imshow("out",image) From 3dc994b263b2363adaadee9ee261c50555ff69c9 Mon Sep 17 00:00:00 2001 From: realdragonturtle <29815169+realdragonturtle@users.noreply.github.com> Date: Wed, 2 Feb 2022 14:44:55 +0100 Subject: [PATCH 02/20] fixed additional gui so that automatic calibration works in steamvr --- bin/mediapipepose.py | 170 +++++++++++++++++++++++++++++-------------- 1 file changed, 117 insertions(+), 53 deletions(-) diff --git a/bin/mediapipepose.py b/bin/mediapipepose.py index b7673e6..19fae1f 100644 --- a/bin/mediapipepose.py +++ b/bin/mediapipepose.py @@ -22,7 +22,7 @@ mp_drawing = mp.solutions.drawing_utils mp_pose = mp.solutions.pose -use_steamvr = False +use_steamvr = True print("Reading parameters...") @@ -75,6 +75,8 @@ def change_recalibrate(): global recalibrate recalibrate = True +global set_rot_y_var, set_rot_x_var, set_rot_z_var, set_scale_var + global_rot_y = R.from_euler('y',0,degrees=True) #default rotations, for 0 degrees around y and x global_rot_x = R.from_euler('x',0,degrees=True) global_rot_z = R.from_euler('z',0,degrees=True) @@ -93,7 +95,8 @@ def rot_change_z(value): posescale = 1 def change_scale(value): global posescale - posescale = value/50 + 0.5 + #posescale = value/50 + 0.5 + posescale = value camera_thread = threading.Thread(target=camera_thread_fun, daemon=True) camera_thread.start() #start our thread, which starts camera capture @@ -101,21 +104,24 @@ def change_scale(value): exit_ready = False def gui_thread_fun(): - global smoothing, calib_rot, calib_tilt, change_recalibrate, rot_change_x, rot_change_y, rot_change_z, global_rot_x, global_rot_y, global_rot_z - - def ready2exit(): - global exit_ready - exit_ready = True - - def update_vals(smoothing_val): - global smoothing - smoothing = smoothing_val + global change_recalibrate, rot_change_x, rot_change_y, rot_change_z, change_scale # functions from main thread to change variables + global smoothing, calib_rot, calib_tilt, calib_scale # variables that are changed in this gui thread but read only in main + global global_rot_x, global_rot_y, global_rot_z, posescale # variables that are read only here and are changed in main + # functions to change above row variables after auto calibration to show on scales vvv + global set_rot_y_var, set_rot_x_var, set_rot_z_var, set_scale_var + + #string_var.trace('w', callback) + def set_rot_y_var(val): + rot_y_var.set(val.as_euler('zyx', degrees=True)[1]) + def set_rot_z_var(val): + rot_z_var.set(val.as_euler('zyx', degrees=True)[0]) + def set_rot_x_var(val): + rot_x_var.set(val.as_euler('zyx', degrees=True)[2]) + def set_scale_var(val): + scale_var.set(val) window = tk.Tk() - frame1 = tk.Frame(window) - frame1.pack() - def show_hide(value, frames): val = value.get() for frame in frames: @@ -126,73 +132,120 @@ def show_hide(value, frames): ### calib rot - varrot = tk.IntVar(value = calib_rot) - rot_check = tk.Checkbutton(frame1, text = "Enable automatic rotation calibration", variable = varrot, command=lambda *args: show_hide(varrot, [rot_y_frame])) - rot_check.pack() - calib_rot = bool(varrot.get()) + def change_rot_auto(): + global calib_rot + calib_rot = calib_rot_var.get() + + frame1 = tk.Frame(window) + frame1.pack() + calib_rot_var = tk.BooleanVar(value=calib_rot) + rot_check = tk.Checkbutton(frame1, text = "Enable automatic rotation calibration", variable = calib_rot_var, command=change_rot_auto)#, command=lambda *args: show_hide(varrot, [rot_y_frame])) rot_y_frame = tk.Frame(frame1) + + rot_check.pack() rot_y_frame.pack() - rot_y = tk.Scale(rot_y_frame, label="Roation y:", from_=0, to=360, command=rot_change_y, orient=tk.HORIZONTAL, - length=800, showvalue=1, tickinterval=50) + rot_y_var = tk.DoubleVar(value=global_rot_y.as_euler('zyx', degrees=True)[1]) + rot_y = tk.Scale(rot_y_frame, label="Roation y:", from_=0, to=360, command=lambda *args: rot_change_y(rot_y_var.get()), orient=tk.HORIZONTAL, + length=400, showvalue=1, tickinterval=60, variable=rot_y_var) rot_y.pack(expand=True,fill='both',side='left') - rot_y.set(global_rot_y.as_euler('zyx', degrees=True)[1]) - tk.Button(rot_y_frame, text="<", command= lambda *args: rot_y.set(rot_y.get()-1), width=10).pack(expand=True,fill='both',side='left') - tk.Button(rot_y_frame, text=">", command= lambda *args: rot_y.set(rot_y.get()+1), width=10).pack(expand=True,fill='both',side='left') + tk.Button(rot_y_frame, text="<", command= lambda *args: rot_y_var.set(rot_y_var.get()-1), width=10).pack(expand=True,fill='both',side='left') + tk.Button(rot_y_frame, text=">", command= lambda *args: rot_y_var.set(rot_y_var.get()+1), width=10).pack(expand=True,fill='both',side='left') separator = ttk.Separator(window, orient='horizontal') separator.pack(fill='x') ## calib tilt + def change_tilt_auto(): + global calib_tilt + calib_tilt = calib_tilt_var.get() + frame2 = tk.Frame(window) frame2.pack() - vartilt = tk.IntVar(value = calib_tilt) - tilt_check = tk.Checkbutton(frame2, text = "Enable automatic tilt calibration", variable = vartilt, command=lambda *args: show_hide(vartilt, [rot_z_frame, rot_x_frame])) + calib_tilt_var = tk.BooleanVar(value=calib_tilt) + tilt_check = tk.Checkbutton(frame2, text="Enable automatic tilt calibration", variable=calib_tilt_var, command=change_tilt_auto)#, command=lambda *args: show_hide(vartilt, [rot_z_frame, rot_x_frame])) tilt_check.pack() - calib_tilt = bool(vartilt.get()) rot_x_frame = tk.Frame(frame2) rot_x_frame.pack() rot_z_frame = tk.Frame(frame2) rot_z_frame.pack() - rot_x = tk.Scale(rot_x_frame, label="Roation x:", from_=90, to=180, command=rot_change_x, orient=tk.HORIZONTAL, - length=800, showvalue=1, tickinterval=30) + rot_x_var = tk.DoubleVar(value=global_rot_x.as_euler('zyx', degrees=True)[2]) + rot_x = tk.Scale(rot_x_frame, label="Roation x:", from_=90, to=180, command=lambda *args: rot_change_x(rot_x_var.get()), orient=tk.HORIZONTAL, + length=400, showvalue=1, tickinterval=15, variable=rot_x_var) rot_x.pack(expand=True,fill='both',side='left') - rot_x.set(global_rot_x.as_euler('zyx', degrees=True)[2]) - tk.Button(rot_x_frame, text="<", command= lambda *args: rot_x.set(rot_x.get()-1), width=10).pack(expand=True,fill='both',side='left') - tk.Button(rot_x_frame, text=">", command= lambda *args: rot_x.set(rot_x.get()+1), width=10).pack(expand=True,fill='both',side='left') + tk.Button(rot_x_frame, text="<", command=lambda *args: rot_x_var.set(rot_x_var.get()-1), width=10).pack(expand=True,fill='both',side='left') + tk.Button(rot_x_frame, text=">", command=lambda *args: rot_x_var.set(rot_x_var.get()+1), width=10).pack(expand=True,fill='both',side='left') - rot_z = tk.Scale(rot_z_frame, label="Roation z:", from_=180, to=360, command=rot_change_z, orient=tk.HORIZONTAL, - length=800, showvalue=1, tickinterval=30) + rot_z_var = tk.DoubleVar(value=global_rot_z.as_euler('zyx', degrees=True)[0]) + rot_z = tk.Scale(rot_z_frame, label="Roation z:", from_=180, to=360, command=lambda *args: rot_change_z(rot_z_var.get()), orient=tk.HORIZONTAL, + length=400, showvalue=1, tickinterval=30, variable=rot_z_var) rot_z.pack(expand=True,fill='both',side='left') - rot_z.set(global_rot_z.as_euler('zyx', degrees=True)[0 ]) - tk.Button(rot_z_frame, text="<", command= lambda *args: rot_z.set(rot_z.get()-1), width=10).pack(expand=True,fill='both',side='left') - tk.Button(rot_z_frame, text=">", command= lambda *args: rot_z.set(rot_z.get()+1), width=10).pack(expand=True,fill='both',side='left') + tk.Button(rot_z_frame, text="<", command=lambda *args: rot_z_var.set(rot_z_var.get()-1), width=10).pack(expand=True,fill='both',side='left') + tk.Button(rot_z_frame, text=">", command=lambda *args: rot_z_var.set(rot_z_var.get()+1), width=10).pack(expand=True,fill='both',side='left') separator = ttk.Separator(window, orient='horizontal') separator.pack(fill='x') + ### calib scale + + def change_scale_auto(): + global calib_scale + calib_scale = calib_scale_var.get() + + frame3 = tk.Frame(window) + frame3.pack() + + calib_scale_var = tk.BooleanVar(value=calib_scale) + scale_check = tk.Checkbutton(frame3, text ="Enable automatic scale calibration", variable=calib_scale_var, command=change_scale_auto)#, command=lambda *args: show_hide(varrot, [rot_y_frame])) + scale_frame = tk.Frame(frame3) - bottom_frame = tk.Frame(window) - bottom_frame.pack() + scale_check.pack() + scale_frame.pack() + + scale_var = tk.DoubleVar(value=posescale) + scale = tk.Scale(scale_frame, label="Scale:", from_=0.5, to=2.0, command=lambda *args: change_scale(scale_var.get()), orient=tk.HORIZONTAL, + length=400, showvalue=1, tickinterval=0.1, variable=scale_var, resolution=0.01) + scale.pack(expand=True,fill='both',side='left') + + tk.Button(scale_frame, text="<", command=lambda *args: scale_var.set(scale_var.get()-0.01), width=10).pack(expand=True,fill='both',side='left') + tk.Button(scale_frame, text=">", command=lambda *args: scale_var.set(scale_var.get()+0.01), width=10).pack(expand=True,fill='both',side='left') + + separator = ttk.Separator(window, orient='horizontal') + separator.pack(fill='x') ## recalibrate - tk.Button(bottom_frame, text='Recalibrate', command=change_recalibrate).pack() + tk.Button(window, text='Recalibrate (automatically recalibrates checked values above)', command=change_recalibrate).pack() + + ## smoothing + + def update_vals(smoothing_val): + global smoothing + smoothing = smoothing_val + + smooth_frame = tk.Frame(window) + smooth_frame.pack() - tk.Label(bottom_frame, text="Smoothing:", width = 50).pack() - smoothingtext = tk.Entry(width = 50) - smoothingtext.pack() + tk.Label(smooth_frame, text="Smoothing:", width = 20).pack(side='left') + smoothingtext = tk.Entry(smooth_frame, width = 10) + smoothingtext.pack(side='left') smoothingtext.insert(0, smoothing) - tk.Button(bottom_frame, text='Update', command=lambda *args: update_vals(float(smoothingtext.get()))).pack() + tk.Button(smooth_frame, text='Update smoothing val', command=lambda *args: update_vals(float(smoothingtext.get()))).pack(side='left') - tk.Button(bottom_frame, text='Press to exit', command=ready2exit).pack() + ## exit + + def ready2exit(): + global exit_ready + exit_ready = True + + tk.Button(window, text='Press to exit', command=ready2exit).pack() window.mainloop() @@ -284,18 +337,21 @@ def sendToSteamVR(text): cv2.namedWindow("out") -if not calib_rot: - cv2.createTrackbar("rotation_y","out",0,360,rot_change_y) #Create rotation sliders and assign callback functions -if not calib_tilt: - cv2.createTrackbar("rotation_x","out",90,180,rot_change_x) - cv2.createTrackbar("rotation_z","out",180,360,rot_change_z) -if not calib_scale: - cv2.createTrackbar("scale","out",25,100,change_scale) -if calib_scale or calib_tilt or calib_rot: - cv2.createTrackbar("recalib","out",0,1,change_recalibrate) +if False: + if not calib_rot: + cv2.createTrackbar("rotation_y","out",0,360,rot_change_y) #Create rotation sliders and assign callback functions + if not calib_tilt: + cv2.createTrackbar("rotation_x","out",90,180,rot_change_x) + cv2.createTrackbar("rotation_z","out",180,360,rot_change_z) + if not calib_scale: + cv2.createTrackbar("scale","out",25,100,change_scale) + if calib_scale or calib_tilt or calib_rot: + cv2.createTrackbar("recalib","out",0,1,change_recalibrate) #Main program loop: + + rotation = 0 i = 0 @@ -413,6 +469,9 @@ def sendToSteamVR(text): value = np.arctan2(feet_middle[2],-feet_middle[1]) print("Postcalib x angle: ", value * 57.295779513) + + set_rot_x_var(global_rot_x) + set_rot_z_var(global_rot_z) if calib_rot: feet_rot = pose3d_og[0] - pose3d_og[5] @@ -432,12 +491,17 @@ def sendToSteamVR(text): value = np.arctan2(feet_rot[0],feet_rot[2]) print("Postcalib y value: ", value * 57.295779513) + + set_rot_y_var(global_rot_y) if calib_scale: #calculate the height of the skeleton, calculate the height in steamvr as distance of hmd from the ground. #divide the two to get scale skelSize = np.max(pose3d_og, axis=0)-np.min(pose3d_og, axis=0) posescale = headsetpos[1]/skelSize[1] + + set_scale_var(posescale) + recalibrate = False else: From 41c40bacd0bc59dbed13f10b8e6c7572c6053eea Mon Sep 17 00:00:00 2001 From: realdragonturtle <29815169+realdragonturtle@users.noreply.github.com> Date: Thu, 3 Feb 2022 17:11:13 +0100 Subject: [PATCH 03/20] created new class for inference gui, organized function calls, moved parameters to its own class --- bin/guitest.py | 6 +- bin/helpers.py | 21 ++- bin/inference_gui.py | 285 +++++++++++++++++++++++++++++ bin/mediapipepose.py | 423 +++++++------------------------------------ bin/parameters.py | 88 +++++++++ 5 files changed, 463 insertions(+), 360 deletions(-) create mode 100644 bin/inference_gui.py create mode 100644 bin/parameters.py diff --git a/bin/guitest.py b/bin/guitest.py index 9d19668..0f848bd 100644 --- a/bin/guitest.py +++ b/bin/guitest.py @@ -26,9 +26,9 @@ def getparams(): if "rotate" not in param: param["rotate"] = None if "camlatency" not in param: - param["camlatency"] = 0.05; + param["camlatency"] = 0.05 if "smooth" not in param: - param["smooth"] = 0.5; + param["smooth"] = 0.5 if "feetrot" not in param: param["feetrot"] = False if "calib_scale" not in param: @@ -165,4 +165,4 @@ def getparams(): return param if __name__ == "__main__": - print(getparams()); + print(getparams()) diff --git a/bin/helpers.py b/bin/helpers.py index 836f103..065d51f 100644 --- a/bin/helpers.py +++ b/bin/helpers.py @@ -291,4 +291,23 @@ def get_rot(pose3d): rot_leg_r = R.from_matrix(leg_r_rot).as_quat() rot_leg_l = R.from_matrix(leg_l_rot).as_quat() - return rot_hip, rot_leg_l, rot_leg_r \ No newline at end of file + return rot_hip, rot_leg_l, rot_leg_r + + +def sendToSteamVR(text): + #Function to send a string to my steamvr driver through a named pipe. + #open pipe -> send string -> read string -> close pipe + #sometimes, something along that pipeline fails for no reason, which is why the try catch is needed. + #returns an array containing the values returned by the driver. + try: + pipe = open(r'\\.\pipe\ApriltagPipeIn', 'rb+', buffering=0) + some_data = str.encode(text) + pipe.write(some_data) + resp = pipe.read(1024) + except: + return ["error"] + string = resp.decode("utf-8") + array = string.split(" ") + pipe.close() + + return array \ No newline at end of file diff --git a/bin/inference_gui.py b/bin/inference_gui.py new file mode 100644 index 0000000..d1bd6a6 --- /dev/null +++ b/bin/inference_gui.py @@ -0,0 +1,285 @@ +import tkinter as tk +from tkinter import ttk +import numpy as np +from scipy.spatial.transform import Rotation as R +import helpers +#import mediapipepose as mep + +class InferenceWindow(tk.Frame): + def __init__(self, root, params, *args, **kwargs): + tk.Frame.__init__(self, root, *args, **kwargs) + + self.params = params + self.root = root + + # calibrate rotation + self.calib_rot_var = tk.BooleanVar(value=self.params.calib_rot) + self.rot_y_var = tk.DoubleVar(value=self.params.global_rot_y.as_euler('zyx', degrees=True)[1]) + + frame1 = tk.Frame(self.root) + frame1.pack() + self.calibrate_rotation_frame(frame1) + + self.put_separator() + + # calibrate tilt + self.calib_tilt_var = tk.BooleanVar(value=self.params.calib_tilt) + self.rot_x_var = tk.DoubleVar(value=self.params.global_rot_x.as_euler('zyx', degrees=True)[2]) + self.rot_z_var = tk.DoubleVar(value=self.params.global_rot_z.as_euler('zyx', degrees=True)[0]) + + frame2 = tk.Frame(self.root) + frame2.pack() + self.calibrate_tilt_frame(frame2) + + self.put_separator() + + # calibrate scale + self.calib_scale_var = tk.BooleanVar(value=self.params.calib_scale) + self.scale_var = tk.DoubleVar(value=self.params.posescale) + + frame3 = tk.Frame(self.root) + frame3.pack() + self.calibrate_scale_frame(frame3) + + self.put_separator() + + # recalibrate + tk.Button(self.root, text='Recalibrate (automatically recalibrates checked values above)', + command=self.autocalibrate).pack() + + # smoothing + frame4 = tk.Frame(self.root) + frame4.pack() + self.change_smooothing_frame(frame4) + + # rotate image + frame5 = tk.Frame(self.root) + frame5.pack() + self.change_image_rotation_frame(frame5) + + # exit + tk.Button(self.root, text='Press to exit', command=self.params.ready2exit).pack() + + #self.root.after(0, self.set_rot_y_var) + #self.root.after(0, self.set_rot_x_var) + + + + def set_rot_y_var(self): + self.rot_y_var.set(self.params.global_rot_y.as_euler('zyx', degrees=True)[1]) + #self.root.after(0, self.set_rot_y_var) + + + def set_rot_z_var(self): + self.rot_z_var.set(self.params.global_rot_z.as_euler('zyx', degrees=True)[0]) + + + def set_rot_x_var(self): + self.rot_x_var.set(self.params.global_rot_x.as_euler('zyx', degrees=True)[2]) + #self.root.after(0, self.set_rot_x_var) + + + def set_scale_var(self): + self.scale_var.set(self.params.posescale) + + + def change_rot_auto(self): + self.params.calib_rot = self.calib_rot_var.get() + print(f"Mark rotation to{'' if self.params.calib_rot else ' NOT'} be automatically calibrated") + + + def calibrate_rotation_frame(self, frame): + rot_check = tk.Checkbutton(frame, text = "Enable automatic rotation calibration", variable = self.calib_rot_var, command=self.change_rot_auto)#, command=lambda *args: show_hide(varrot, [rot_y_frame])) + rot_y_frame = tk.Frame(frame) + + rot_check.pack() + rot_y_frame.pack() + + rot_y = tk.Scale(rot_y_frame, label="Roation y:", from_=0, to=360, #command=lambda *args: self.params.rot_change_y(self.rot_y_var.get()), + orient=tk.HORIZONTAL, length=400, showvalue=1, tickinterval=60, variable=self.rot_y_var) + rot_y.pack(expand=True,fill='both',side='left') + + self.rot_y_var.trace_add('write', callback=lambda var, index, mode: self.params.rot_change_y(self.rot_y_var.get())) + + tk.Button(rot_y_frame, text="<", command= lambda *args: self.rot_y_var.set(self.rot_y_var.get()-1), width=10).pack(expand=True,fill='both',side='left') + tk.Button(rot_y_frame, text=">", command= lambda *args: self.rot_y_var.set(self.rot_y_var.get()+1), width=10).pack(expand=True,fill='both',side='left') + + + def change_tilt_auto(self): + self.params.calib_tilt = self.calib_tilt_var.get() + print(f"Mark tilt to{'' if self.params.calib_tilt else ' NOT'} be automatically calibrated") + + + def calibrate_tilt_frame(self, frame): + + tilt_check = tk.Checkbutton(frame, text="Enable automatic tilt calibration", variable=self.calib_tilt_var, command=self.change_tilt_auto)#, command=lambda *args: show_hide(vartilt, [rot_z_frame, rot_x_frame])) + tilt_check.pack() + + rot_x_frame = tk.Frame(frame) + rot_x_frame.pack() + rot_z_frame = tk.Frame(frame) + rot_z_frame.pack() + + rot_x = tk.Scale(rot_x_frame, label="Roation x:", from_=90, to=180, #command=lambda *args: self.params.rot_change_x(self.rot_x_var.get()), + orient=tk.HORIZONTAL, length=400, showvalue=1, tickinterval=15, variable=self.rot_x_var) + rot_x.pack(expand=True,fill='both',side='left') + self.rot_x_var.trace_add('write', callback=lambda var, index, mode: self.params.rot_change_x(self.rot_x_var.get())) + + tk.Button(rot_x_frame, text="<", command=lambda *args: self.rot_x_var.set(self.rot_x_var.get()-1), width=10).pack(expand=True,fill='both',side='left') + tk.Button(rot_x_frame, text=">", command=lambda *args: self.rot_x_var.set(self.rot_x_var.get()+1), width=10).pack(expand=True,fill='both',side='left') + + rot_z = tk.Scale(rot_z_frame, label="Roation z:", from_=180, to=360, #command=lambda *args: self.params.rot_change_z(self.rot_z_var.get()), + orient=tk.HORIZONTAL, length=400, showvalue=1, tickinterval=30, variable=self.rot_z_var) + rot_z.pack(expand=True,fill='both',side='left') + self.rot_z_var.trace_add('write', callback=lambda var, index, mode: self.params.rot_change_z(self.rot_z_var.get())) + + tk.Button(rot_z_frame, text="<", command=lambda *args: self.rot_z_var.set(self.rot_z_var.get()-1), width=10).pack(expand=True,fill='both',side='left') + tk.Button(rot_z_frame, text=">", command=lambda *args: self.rot_z_var.set(self.rot_z_var.get()+1), width=10).pack(expand=True,fill='both',side='left') + + + def change_scale_auto(self): + self.params.calib_scale = self.calib_scale_var.get() + print(f"Mark scale to{'' if self.params.calib_scale else ' NOT'} be automatically calibrated") + + + def calibrate_scale_frame(self, frame): + + scale_check = tk.Checkbutton(frame, text ="Enable automatic scale calibration", variable=self.calib_scale_var, command=self.change_scale_auto)#, command=lambda *args: show_hide(varrot, [rot_y_frame])) + scale_frame = tk.Frame(frame) + + scale_check.pack() + scale_frame.pack() + + scale = tk.Scale(scale_frame, label="Scale:", from_=0.5, to=2.0, #command=lambda *args: self.params.change_scale(self.scale_var.get()), + orient=tk.HORIZONTAL, length=400, showvalue=1, tickinterval=0.1, variable=self.scale_var, resolution=0.01) + scale.pack(expand=True,fill='both',side='left') + self.scale_var.trace_add('write', callback=lambda var, index, mode: self.params.change_scale(self.scale_var.get())) + + tk.Button(scale_frame, text="<", command=lambda *args: self.scale_var.set(self.scale_var.get()-0.01), width=10).pack(expand=True,fill='both',side='left') + tk.Button(scale_frame, text=">", command=lambda *args: self.scale_var.set(self.scale_var.get()+0.01), width=10).pack(expand=True,fill='both',side='left') + + + def change_smooothing_frame(self, frame): + + tk.Label(frame, text="Smoothing:", width = 20).pack(side='left') + smoothingtext = tk.Entry(frame, width = 10) + smoothingtext.pack(side='left') + smoothingtext.insert(0, self.params.smoothing) + + tk.Button(frame, text='Update smoothing value', command=lambda *args: self.params.change_smoothing(float(smoothingtext.get()))).pack(side='left') + + + def change_image_rotation_frame(self, frame): + rot_img_var = tk.IntVar(value=0)#rotate_image) + tk.Label(frame, text="Image rotation clockwise:", width = 20).grid(row=0, column=0) + tk.Radiobutton(frame, text="0", variable = rot_img_var, value = 0).grid(row=0, column=1) + tk.Radiobutton(frame, text="90", variable = rot_img_var, value = 1).grid(row=0, column=2) + tk.Radiobutton(frame, text="180", variable = rot_img_var, value = 2).grid(row=0, column=3) + tk.Radiobutton(frame, text="270", variable = rot_img_var, value = 3).grid(row=0, column=4) + + rot_img_var.trace_add('write', callback=lambda var, index, mode: self.params.change_img_rot(rot_img_var.get())) + + + def autocalibrate(self): + + for _ in range(10): + array = helpers.sendToSteamVR("getdevicepose 0") #get hmd data to allign our skeleton to + + if "error" in array: #continue to next iteration if there is an error + continue + else: + break + + if "error" in array: #continue to next iteration if there is an error + print("Failed to contact SteamVR after 10 tries... Try to autocalibrate again.") + return + + headsetpos = [float(array[3]),float(array[4]),float(array[5])] + headsetrot = R.from_quat([float(array[7]),float(array[8]),float(array[9]),float(array[6])]) + + neckoffset = headsetrot.apply(self.params.hmd_to_neck_offset) #the neck position seems to be the best point to allign to, as its well defined on + #the skeleton (unlike the eyes/nose, which jump around) and can be calculated from hmd. + + if self.params.calib_tilt: + feet_middle = (self.params.pose3d_og[0] + self.params.pose3d_og[5])/2 + + print(feet_middle) + + ## roll calibaration + + value = np.arctan2(feet_middle[0],-feet_middle[1]) + + print("Precalib z angle: ", value * 57.295779513) + + self.params.global_rot_z = R.from_euler('z',-value) + self.set_rot_z_var() + + for j in range(self.params.pose3d_og.shape[0]): + self.params.pose3d_og[j] = self.params.global_rot_z.apply(self.params.pose3d_og[j]) + + feet_middle = (self.params.pose3d_og[0] + self.params.pose3d_og[5])/2 + value = np.arctan2(feet_middle[0],-feet_middle[1]) + + print("Postcalib z angle: ", value * 57.295779513) + + ##tilt calibration + + value = np.arctan2(feet_middle[2],-feet_middle[1]) + + print("Precalib x angle: ", value * 57.295779513) + + self.params.global_rot_x = R.from_euler('x',value) + self.set_rot_x_var() + + for j in range(self.params.pose3d_og.shape[0]): + self.params.pose3d_og[j] = self.params.global_rot_x.apply(self.params.pose3d_og[j]) + + feet_middle = (self.params.pose3d_og[0] + self.params.pose3d_og[5])/2 + value = np.arctan2(feet_middle[2],-feet_middle[1]) + + print("Postcalib x angle: ", value * 57.295779513) + + if self.params.calib_rot: + feet_rot = self.params.pose3d_og[0] - self.params.pose3d_og[5] + value = np.arctan2(feet_rot[0],feet_rot[2]) + value_hmd = np.arctan2(headsetrot.as_matrix()[0][0],headsetrot.as_matrix()[2][0]) + print("Precalib y value: ", value * 57.295779513) + print("hmd y value: ", value_hmd * 57.295779513) + + value = value - value_hmd + + self.params.global_rot_y = R.from_euler('y',-value) + self.set_rot_y_var() + + for j in range(self.params.pose3d_og.shape[0]): + self.params.pose3d_og[j] = self.params.global_rot_y.apply(self.params.pose3d_og[j]) + + feet_rot = self.params.pose3d_og[0] - self.params.pose3d_og[5] + value = np.arctan2(feet_rot[0],feet_rot[2]) + + print("Postcalib y value: ", value * 57.295779513) + + if self.params.calib_scale: + #calculate the height of the skeleton, calculate the height in steamvr as distance of hmd from the ground. + #divide the two to get scale + skelSize = np.max(self.params.pose3d_og, axis=0)-np.min(self.params.pose3d_og, axis=0) + self.params.posescale = headsetpos[1]/skelSize[1] + + self.set_scale_var() + + self.params.recalibrate = False + + + def put_separator(self): + separator = ttk.Separator(self.root, orient='horizontal') + separator.pack(fill='x') + + +def make_inference_gui(_params): + root = tk.Tk() + InferenceWindow(root, _params).pack(side="top", fill="both", expand=True) + root.mainloop() + +if __name__ == "__main__": + #make_inference_gui() + print("hehe") \ No newline at end of file diff --git a/bin/mediapipepose.py b/bin/mediapipepose.py index 19fae1f..034d9bb 100644 --- a/bin/mediapipepose.py +++ b/bin/mediapipepose.py @@ -14,9 +14,11 @@ import threading import cv2 import numpy as np -from helpers import mediapipeTo3dpose, get_rot_mediapipe, get_rot_hands, draw_pose, keypoints_to_original, normalize_screen_coordinates, get_rot +from helpers import mediapipeTo3dpose, get_rot_mediapipe, get_rot_hands, draw_pose, keypoints_to_original, normalize_screen_coordinates, get_rot, sendToSteamVR from scipy.spatial.transform import Rotation as R -from guitest import getparams + +import inference_gui +import parameters import mediapipe as mp mp_drawing = mp.solutions.drawing_utils @@ -26,27 +28,7 @@ print("Reading parameters...") -param = getparams() - -#PARAMETERS: -#model = 1 #TODO: add parameter for which model size to use -maximgsize = param["imgsize"] #to prevent working with huge images, images that have one axis larger than this value will be downscaled. -cameraid = param["camid"] #to use with an usb or virtual webcam. If 0 doesnt work/opens wrong camera, try numbers 1-5 or so -#cameraid = "http://192.168.1.102:8080/video" #to use ip webcam, uncomment this line and change to your ip -hmd_to_neck_offset = param["neckoffset"] #offset of your hmd to the base of your neck, to ensure the tracking is stable even if you look around. Default is 20cm down, 10cm back. -preview_skeleton = param["prevskel"] #if True, whole skeleton will appear in vr 2 meters in front of you. Good to visualize if everything is working -dont_wait_hmd = param["waithmd"] #dont wait for movement from hmd, start inference immediately. -rotate_image = param["rotate"] # cv2.ROTATE_90_CLOCKWISE # cv2.ROTATE_90_COUTERCLOCKWISE # cv2.ROTATE_180 # None # if you want, rotate the camera -camera_latency = param["camlatency"] -smoothing = param["smooth"] -feet_rotation = param["feetrot"] -calib_scale = param["calib_scale"] -calib_tilt = param["calib_tilt"] -calib_rot = param["calib_rot"] -use_hands = param["use_hands"] -ignore_hip = param["ignore_hip"] - -prev_smoothing = smoothing +params = parameters.Parameters() print("Opening camera...") @@ -58,8 +40,10 @@ def camera_thread_fun(): #Mostly used to free the main thread from image decoding overhead and to ensure frames are just skipped if detection is slower than camera fps global cameraid, image_from_thread, image_ready - if len(cameraid) <= 2: - cameraid = int(cameraid) + if len(params.cameraid) <= 2: + cameraid = int(params.cameraid) + else: + cameraid = params.cameraid cap = cv2.VideoCapture(cameraid) print("Camera opened!") @@ -70,206 +54,13 @@ def camera_thread_fun(): assert ret, "Camera capture failed! Check the cameraid parameter." -recalibrate = False -def change_recalibrate(): - global recalibrate - recalibrate = True - -global set_rot_y_var, set_rot_x_var, set_rot_z_var, set_scale_var - -global_rot_y = R.from_euler('y',0,degrees=True) #default rotations, for 0 degrees around y and x -global_rot_x = R.from_euler('x',0,degrees=True) -global_rot_z = R.from_euler('z',0,degrees=True) -def rot_change_y(value): - global global_rot_y #callback functions. Whenever the value on sliders are changed, they are called - global_rot_y = R.from_euler('y',value,degrees=True) #and the rotation is updated with the new value. - -def rot_change_x(value): - global global_rot_x - global_rot_x = R.from_euler('x',value-90,degrees=True) - -def rot_change_z(value): - global global_rot_z - global_rot_z = R.from_euler('z',value-180,degrees=True) - -posescale = 1 -def change_scale(value): - global posescale - #posescale = value/50 + 0.5 - posescale = value - camera_thread = threading.Thread(target=camera_thread_fun, daemon=True) camera_thread.start() #start our thread, which starts camera capture -exit_ready = False - -def gui_thread_fun(): - global change_recalibrate, rot_change_x, rot_change_y, rot_change_z, change_scale # functions from main thread to change variables - global smoothing, calib_rot, calib_tilt, calib_scale # variables that are changed in this gui thread but read only in main - global global_rot_x, global_rot_y, global_rot_z, posescale # variables that are read only here and are changed in main - # functions to change above row variables after auto calibration to show on scales vvv - global set_rot_y_var, set_rot_x_var, set_rot_z_var, set_scale_var - - #string_var.trace('w', callback) - def set_rot_y_var(val): - rot_y_var.set(val.as_euler('zyx', degrees=True)[1]) - def set_rot_z_var(val): - rot_z_var.set(val.as_euler('zyx', degrees=True)[0]) - def set_rot_x_var(val): - rot_x_var.set(val.as_euler('zyx', degrees=True)[2]) - def set_scale_var(val): - scale_var.set(val) - - window = tk.Tk() - - def show_hide(value, frames): - val = value.get() - for frame in frames: - if not val: - frame.pack() #.grid(row=1) - else: - frame.pack_forget() # .grid_forget() - - ### calib rot - - def change_rot_auto(): - global calib_rot - calib_rot = calib_rot_var.get() - - frame1 = tk.Frame(window) - frame1.pack() - - calib_rot_var = tk.BooleanVar(value=calib_rot) - rot_check = tk.Checkbutton(frame1, text = "Enable automatic rotation calibration", variable = calib_rot_var, command=change_rot_auto)#, command=lambda *args: show_hide(varrot, [rot_y_frame])) - rot_y_frame = tk.Frame(frame1) - - rot_check.pack() - rot_y_frame.pack() - - rot_y_var = tk.DoubleVar(value=global_rot_y.as_euler('zyx', degrees=True)[1]) - rot_y = tk.Scale(rot_y_frame, label="Roation y:", from_=0, to=360, command=lambda *args: rot_change_y(rot_y_var.get()), orient=tk.HORIZONTAL, - length=400, showvalue=1, tickinterval=60, variable=rot_y_var) - rot_y.pack(expand=True,fill='both',side='left') - - tk.Button(rot_y_frame, text="<", command= lambda *args: rot_y_var.set(rot_y_var.get()-1), width=10).pack(expand=True,fill='both',side='left') - tk.Button(rot_y_frame, text=">", command= lambda *args: rot_y_var.set(rot_y_var.get()+1), width=10).pack(expand=True,fill='both',side='left') - - separator = ttk.Separator(window, orient='horizontal') - separator.pack(fill='x') - - ## calib tilt - - def change_tilt_auto(): - global calib_tilt - calib_tilt = calib_tilt_var.get() - - frame2 = tk.Frame(window) - frame2.pack() - - calib_tilt_var = tk.BooleanVar(value=calib_tilt) - tilt_check = tk.Checkbutton(frame2, text="Enable automatic tilt calibration", variable=calib_tilt_var, command=change_tilt_auto)#, command=lambda *args: show_hide(vartilt, [rot_z_frame, rot_x_frame])) - tilt_check.pack() - - rot_x_frame = tk.Frame(frame2) - rot_x_frame.pack() - rot_z_frame = tk.Frame(frame2) - rot_z_frame.pack() - - rot_x_var = tk.DoubleVar(value=global_rot_x.as_euler('zyx', degrees=True)[2]) - rot_x = tk.Scale(rot_x_frame, label="Roation x:", from_=90, to=180, command=lambda *args: rot_change_x(rot_x_var.get()), orient=tk.HORIZONTAL, - length=400, showvalue=1, tickinterval=15, variable=rot_x_var) - rot_x.pack(expand=True,fill='both',side='left') - tk.Button(rot_x_frame, text="<", command=lambda *args: rot_x_var.set(rot_x_var.get()-1), width=10).pack(expand=True,fill='both',side='left') - tk.Button(rot_x_frame, text=">", command=lambda *args: rot_x_var.set(rot_x_var.get()+1), width=10).pack(expand=True,fill='both',side='left') - - rot_z_var = tk.DoubleVar(value=global_rot_z.as_euler('zyx', degrees=True)[0]) - rot_z = tk.Scale(rot_z_frame, label="Roation z:", from_=180, to=360, command=lambda *args: rot_change_z(rot_z_var.get()), orient=tk.HORIZONTAL, - length=400, showvalue=1, tickinterval=30, variable=rot_z_var) - rot_z.pack(expand=True,fill='both',side='left') - tk.Button(rot_z_frame, text="<", command=lambda *args: rot_z_var.set(rot_z_var.get()-1), width=10).pack(expand=True,fill='both',side='left') - tk.Button(rot_z_frame, text=">", command=lambda *args: rot_z_var.set(rot_z_var.get()+1), width=10).pack(expand=True,fill='both',side='left') - - separator = ttk.Separator(window, orient='horizontal') - separator.pack(fill='x') - - ### calib scale - - def change_scale_auto(): - global calib_scale - calib_scale = calib_scale_var.get() - - frame3 = tk.Frame(window) - frame3.pack() - - calib_scale_var = tk.BooleanVar(value=calib_scale) - scale_check = tk.Checkbutton(frame3, text ="Enable automatic scale calibration", variable=calib_scale_var, command=change_scale_auto)#, command=lambda *args: show_hide(varrot, [rot_y_frame])) - scale_frame = tk.Frame(frame3) - - scale_check.pack() - scale_frame.pack() - - scale_var = tk.DoubleVar(value=posescale) - scale = tk.Scale(scale_frame, label="Scale:", from_=0.5, to=2.0, command=lambda *args: change_scale(scale_var.get()), orient=tk.HORIZONTAL, - length=400, showvalue=1, tickinterval=0.1, variable=scale_var, resolution=0.01) - scale.pack(expand=True,fill='both',side='left') - - tk.Button(scale_frame, text="<", command=lambda *args: scale_var.set(scale_var.get()-0.01), width=10).pack(expand=True,fill='both',side='left') - tk.Button(scale_frame, text=">", command=lambda *args: scale_var.set(scale_var.get()+0.01), width=10).pack(expand=True,fill='both',side='left') - - separator = ttk.Separator(window, orient='horizontal') - separator.pack(fill='x') - - ## recalibrate - - tk.Button(window, text='Recalibrate (automatically recalibrates checked values above)', command=change_recalibrate).pack() - - ## smoothing - - def update_vals(smoothing_val): - global smoothing - smoothing = smoothing_val - - smooth_frame = tk.Frame(window) - smooth_frame.pack() - - tk.Label(smooth_frame, text="Smoothing:", width = 20).pack(side='left') - smoothingtext = tk.Entry(smooth_frame, width = 10) - smoothingtext.pack(side='left') - smoothingtext.insert(0, smoothing) - - tk.Button(smooth_frame, text='Update smoothing val', command=lambda *args: update_vals(float(smoothingtext.get()))).pack(side='left') - - ## exit - - def ready2exit(): - global exit_ready - exit_ready = True - - tk.Button(window, text='Press to exit', command=ready2exit).pack() - - window.mainloop() - -gui_thread = threading.Thread(target=gui_thread_fun, daemon=True) +gui_thread = threading.Thread(target=inference_gui.make_inference_gui, args=(params,), daemon=True) gui_thread.start() -def sendToSteamVR(text): - #Function to send a string to my steamvr driver through a named pipe. - #open pipe -> send string -> read string -> close pipe - #sometimes, something along that pipeline fails for no reason, which is why the try catch is needed. - #returns an array containing the values returned by the driver. - try: - pipe = open(r'\\.\pipe\ApriltagPipeIn', 'rb+', buffering=0) - some_data = str.encode(text) - pipe.write(some_data) - resp = pipe.read(1024) - except: - return ["error"] - string = resp.decode("utf-8") - array = string.split(" ") - pipe.close() - - return array - + if use_steamvr: print("Connecting to SteamVR") @@ -292,18 +83,18 @@ def sendToSteamVR(text): numtrackers = int(numtrackers[2]) #games use 3 trackers, but we can also send the entire skeleton if we want to look at how it works -totaltrackers = 23 if preview_skeleton else 3 -if use_hands: +totaltrackers = 23 if params.preview_skeleton else 3 +if params.use_hands: totaltrackers = 5 -if ignore_hip: +if params.ignore_hip: totaltrackers -= 1 roles = ["TrackerRole_Waist", "TrackerRole_RightFoot", "TrackerRole_LeftFoot"] -if ignore_hip and not preview_skeleton: +if params.ignore_hip and not params.preview_skeleton: del roles[0] -if(use_hands): +if params.use_hands: roles.append("TrackerRole_Handed") roles.append("TrackerRole_Handed") @@ -320,9 +111,9 @@ def sendToSteamVR(text): time.sleep(0.2) time.sleep(0.2) - resp = sendToSteamVR(f"settings 50 {smoothing}") + resp = sendToSteamVR(f"settings 50 {params.smoothing}") while "error" in resp: - resp = sendToSteamVR(f"settings 50 {smoothing}") + resp = sendToSteamVR(f"settings 50 {params.smoothing}") print(resp) time.sleep(1) @@ -335,41 +126,28 @@ def sendToSteamVR(text): model_complexity=1) - cv2.namedWindow("out") -if False: - if not calib_rot: - cv2.createTrackbar("rotation_y","out",0,360,rot_change_y) #Create rotation sliders and assign callback functions - if not calib_tilt: - cv2.createTrackbar("rotation_x","out",90,180,rot_change_x) - cv2.createTrackbar("rotation_z","out",180,360,rot_change_z) - if not calib_scale: - cv2.createTrackbar("scale","out",25,100,change_scale) - if calib_scale or calib_tilt or calib_rot: - cv2.createTrackbar("recalib","out",0,1,change_recalibrate) - -#Main program loop: - +#Main program loop: rotation = 0 i = 0 while(True): # Capture frame-by-frame - if exit_ready: + if params.exit_ready: cv2.destroyAllWindows() print("Exiting... You can close the window after 10 seconds.") exit(0) - if prev_smoothing != smoothing: - print(f"Changed smoothing value from {prev_smoothing} to {smoothing}") - prev_smoothing = smoothing + if params.prev_smoothing != params.smoothing: + print(f"Changed smoothing value from {params.prev_smoothing} to {params.smoothing}") + params.prev_smoothing = params.smoothing if use_steamvr: - resp = sendToSteamVR(f"settings 50 {smoothing}") + resp = sendToSteamVR(f"settings 50 {params.smoothing}") while "error" in resp: - resp = sendToSteamVR(f"settings 50 {smoothing}") + resp = sendToSteamVR(f"settings 50 {params.smoothing}") print(resp) time.sleep(1) @@ -380,11 +158,11 @@ def sendToSteamVR(text): img = image_from_thread.copy() #some may say I need a mutex here. I say this works fine. image_ready = False - if rotate_image is not None: #if set, rotate the image - img = cv2.rotate(img, rotate_image) + if params.rotate_image is not None: #if set, rotate the image + img = cv2.rotate(img, params.rotate_image) - if max(img.shape) > maximgsize: #if set, ensure image does not exceed the given size. - ratio = max(img.shape)/maximgsize + if max(img.shape) > params.maximgsize: #if set, ensure image does not exceed the given size. + ratio = max(img.shape)/params.maximgsize img = cv2.resize(img,(int(img.shape[1]/ratio),int(img.shape[0]/ratio))) img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB) @@ -406,18 +184,19 @@ def sendToSteamVR(text): pose3d[:,1] = -pose3d[:,1] pose3d_og = pose3d.copy() + params.pose3d_og = pose3d_og for j in range(pose3d.shape[0]): #apply the rotations from the sliders - pose3d[j] = global_rot_z.apply(pose3d[j]) - pose3d[j] = global_rot_x.apply(pose3d[j]) - pose3d[j] = global_rot_y.apply(pose3d[j]) + pose3d[j] = params.global_rot_z.apply(pose3d[j]) + pose3d[j] = params.global_rot_x.apply(pose3d[j]) + pose3d[j] = params.global_rot_y.apply(pose3d[j]) - if not feet_rotation: + if not params.feet_rotation: rots = get_rot(pose3d) #get rotation data of feet and hips from the position-only skeleton data else: rots = get_rot_mediapipe(pose3d) - if use_hands: + if params.use_hands: hand_rots = get_rot_hands(pose3d) if use_steamvr: @@ -429,113 +208,44 @@ def sendToSteamVR(text): headsetpos = [float(array[3]),float(array[4]),float(array[5])] headsetrot = R.from_quat([float(array[7]),float(array[8]),float(array[9]),float(array[6])]) - neckoffset = headsetrot.apply(hmd_to_neck_offset) #the neck position seems to be the best point to allign to, as its well defined on + neckoffset = headsetrot.apply(params.hmd_to_neck_offset) #the neck position seems to be the best point to allign to, as its well defined on #the skeleton (unlike the eyes/nose, which jump around) and can be calculated from hmd. - if recalibrate: - - if calib_tilt: - feet_middle = (pose3d_og[0] + pose3d_og[5])/2 - - print(feet_middle) - - ## roll calibaration - - value = np.arctan2(feet_middle[0],-feet_middle[1]) - - print("Precalib z angle: ", value * 57.295779513) - - global_rot_z = R.from_euler('z',-value) - - for j in range(pose3d_og.shape[0]): - pose3d_og[j] = global_rot_z.apply(pose3d_og[j]) - - feet_middle = (pose3d_og[0] + pose3d_og[5])/2 - value = np.arctan2(feet_middle[0],-feet_middle[1]) - - print("Postcalib z angle: ", value * 57.295779513) - - ##tilt calibration - - value = np.arctan2(feet_middle[2],-feet_middle[1]) - - print("Precalib x angle: ", value * 57.295779513) - - global_rot_x = R.from_euler('x',value) - - for j in range(pose3d_og.shape[0]): - pose3d_og[j] = global_rot_x.apply(pose3d_og[j]) - - feet_middle = (pose3d_og[0] + pose3d_og[5])/2 - value = np.arctan2(feet_middle[2],-feet_middle[1]) - - print("Postcalib x angle: ", value * 57.295779513) - set_rot_x_var(global_rot_x) - set_rot_z_var(global_rot_z) - - if calib_rot: - feet_rot = pose3d_og[0] - pose3d_og[5] - value = np.arctan2(feet_rot[0],feet_rot[2]) - value_hmd = np.arctan2(headsetrot.as_matrix()[0][0],headsetrot.as_matrix()[2][0]) - print("Precalib y value: ", value * 57.295779513) - print("hmd y value: ", value_hmd * 57.295779513) - - value = value - value_hmd - - global_rot_y = R.from_euler('y',-value) - - for j in range(pose3d_og.shape[0]): - pose3d_og[j] = global_rot_y.apply(pose3d_og[j]) - - feet_rot = pose3d_og[0] - pose3d_og[5] - value = np.arctan2(feet_rot[0],feet_rot[2]) - - print("Postcalib y value: ", value * 57.295779513) - - set_rot_y_var(global_rot_y) - - if calib_scale: - #calculate the height of the skeleton, calculate the height in steamvr as distance of hmd from the ground. - #divide the two to get scale - skelSize = np.max(pose3d_og, axis=0)-np.min(pose3d_og, axis=0) - posescale = headsetpos[1]/skelSize[1] - - set_scale_var(posescale) - - recalibrate = False + if params.recalibrate: + print("frame to recalibrate") else: - pose3d = pose3d * posescale #rescale skeleton to calibrated height - #print(pose3d) - offset = pose3d[7] - (headsetpos+neckoffset) #calculate the position of the skeleton - if not preview_skeleton: - numadded = 3 - if not ignore_hip: - for i in [(0,1),(5,2),(6,0)]: - joint = pose3d[i[0]] - offset #for each foot and hips, offset it by skeleton position and send to steamvr - if use_steamvr: - sendToSteamVR(f"updatepose {i[1]} {joint[0]} {joint[1]} {joint[2]} {rots[i[1]][3]} {rots[i[1]][0]} {rots[i[1]][1]} {rots[i[1]][2]} {camera_latency} 0.8") - else: - for i in [(0,0),(5,1)]: - joint = pose3d[i[0]] - offset #for each foot and hips, offset it by skeleton position and send to steamvr - if use_steamvr: - sendToSteamVR(f"updatepose {i[1]} {joint[0]} {joint[1]} {joint[2]} {rots[i[1]][3]} {rots[i[1]][0]} {rots[i[1]][1]} {rots[i[1]][2]} {camera_latency} 0.8") - numadded = 2 - if use_hands: - for i in [(10,0),(15,1)]: - joint = pose3d[i[0]] - offset #for each foot and hips, offset it by skeleton position and send to steamvr - if use_steamvr: - sendToSteamVR(f"updatepose {i[1]+numadded} {joint[0]} {joint[1]} {joint[2]} {hand_rots[i[1]][3]} {hand_rots[i[1]][0]} {hand_rots[i[1]][1]} {hand_rots[i[1]][2]} {camera_latency} 0.8") - - + pose3d = pose3d * params.posescale #rescale skeleton to calibrated height + #print(pose3d) + offset = pose3d[7] - (headsetpos+neckoffset) #calculate the position of the skeleton + if not params.preview_skeleton: + numadded = 3 + if not params.ignore_hip: + for i in [(0,1),(5,2),(6,0)]: + joint = pose3d[i[0]] - offset #for each foot and hips, offset it by skeleton position and send to steamvr + if use_steamvr: + sendToSteamVR(f"updatepose {i[1]} {joint[0]} {joint[1]} {joint[2]} {rots[i[1]][3]} {rots[i[1]][0]} {rots[i[1]][1]} {rots[i[1]][2]} {params.camera_latency} 0.8") else: - for i in range(23): - joint = pose3d[i] - offset #if previewing skeleton, send the position of each keypoint to steamvr without rotation + for i in [(0,0),(5,1)]: + joint = pose3d[i[0]] - offset #for each foot and hips, offset it by skeleton position and send to steamvr if use_steamvr: - sendToSteamVR(f"updatepose {i} {joint[0]} {joint[1]} {joint[2] - 2} 1 0 0 0 {camera_latency} 0.8") - - print(f"Inference time: {time.time()-t0}\nSmoothing value: {smoothing}\n") #print how long it took to detect and calculate everything + sendToSteamVR(f"updatepose {i[1]} {joint[0]} {joint[1]} {joint[2]} {rots[i[1]][3]} {rots[i[1]][0]} {rots[i[1]][1]} {rots[i[1]][2]} {params.camera_latency} 0.8") + numadded = 2 + if params.use_hands: + for i in [(10,0),(15,1)]: + joint = pose3d[i[0]] - offset #for each foot and hips, offset it by skeleton position and send to steamvr + if use_steamvr: + sendToSteamVR(f"updatepose {i[1]+numadded} {joint[0]} {joint[1]} {joint[2]} {hand_rots[i[1]][3]} {hand_rots[i[1]][0]} {hand_rots[i[1]][1]} {hand_rots[i[1]][2]} {params.camera_latency} 0.8") + + + else: + for i in range(23): + joint = pose3d[i] - offset #if previewing skeleton, send the position of each keypoint to steamvr without rotation + if use_steamvr: + sendToSteamVR(f"updatepose {i} {joint[0]} {joint[1]} {joint[2] - 2} 1 0 0 0 {params.camera_latency} 0.8") + #print(f"Inference time: {time.time()-t0}\nSmoothing value: {smoothing}\n") #print how long it took to detect and calculate everything + inference_time = time.time() - t0 #cv2.imshow("out",image) #cv2.waitKey(0) @@ -543,6 +253,7 @@ def sendToSteamVR(text): img = cv2.cvtColor(img,cv2.COLOR_RGB2BGR) #convert back to bgr and draw the pose mp_drawing.draw_landmarks( img, results.pose_landmarks, mp_pose.POSE_CONNECTIONS) + img = cv2.putText(img, f"{inference_time:1.3f}", (10,30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 1, cv2.LINE_AA) cv2.imshow("out",img) #show image, exit program if we press esc if cv2.waitKey(1) == 27: diff --git a/bin/parameters.py b/bin/parameters.py new file mode 100644 index 0000000..ee1ca19 --- /dev/null +++ b/bin/parameters.py @@ -0,0 +1,88 @@ +from guitest import getparams +from scipy.spatial.transform import Rotation as R +import cv2 +import tkinter as tk + +class Parameters(): + def __init__(self) -> None: + param = getparams() + + #PARAMETERS: + #model = 1 #TODO: add parameter for which model size to use + self.maximgsize = param["imgsize"] #to prevent working with huge images, images that have one axis larger than this value will be downscaled. + self.cameraid = param["camid"] #to use with an usb or virtual webcam. If 0 doesnt work/opens wrong camera, try numbers 1-5 or so + #cameraid = "http://192.168.1.102:8080/video" #to use ip webcam, uncomment this line and change to your ip + self.hmd_to_neck_offset = param["neckoffset"] #offset of your hmd to the base of your neck, to ensure the tracking is stable even if you look around. Default is 20cm down, 10cm back. + self.preview_skeleton = param["prevskel"] #if True, whole skeleton will appear in vr 2 meters in front of you. Good to visualize if everything is working + self.dont_wait_hmd = param["waithmd"] #dont wait for movement from hmd, start inference immediately. + self.rotate_image = param["rotate"] # cv2.ROTATE_90_CLOCKWISE # cv2.ROTATE_90_COUTERCLOCKWISE # cv2.ROTATE_180 # None # if you want, rotate the camera + self.camera_latency = param["camlatency"] + self.smoothing = param["smooth"] + self.feet_rotation = param["feetrot"] + self.calib_scale = param["calib_scale"] + self.calib_tilt = param["calib_tilt"] + self.calib_rot = param["calib_rot"] + self.use_hands = param["use_hands"] + self.ignore_hip = param["ignore_hip"] + + self.prev_smoothing = self.smoothing + + self.recalibrate = False + + self.global_rot_y = R.from_euler('y',0,degrees=True) #default rotations, for 0 degrees around y and x + self.global_rot_x = R.from_euler('x',0,degrees=True) + self.global_rot_z = R.from_euler('z',0,degrees=True) + + self.posescale = 1 + + self.exit_ready = False + + self.img_rot_dict = {0: None, 1: cv2.ROTATE_90_CLOCKWISE, 2: cv2.ROTATE_180, 3: cv2.ROTATE_90_COUNTERCLOCKWISE} + + # trace variables + + #self.root = tk.Tk() + #self.rot_y_var = tk.DoubleVar(value=self.global_rot_y.as_euler('zyx', degrees=True)[1]) + + + + def change_recalibrate(self): + self.recalibrate = True + + + def rot_change_y(self, value): #callback functions. Whenever the value on sliders are changed, they are called + print(f"Changed y rotation value to {value}") + self.global_rot_y = R.from_euler('y',value,degrees=True) #and the rotation is updated with the new value. + + + def rot_change_x(self, value): + print(f"Changed x rotation value to {value}") + self.global_rot_x = R.from_euler('x',value-90,degrees=True) + + def rot_change_z(self, value): + print(f"Changed z rotation value to {value}") + self.global_rot_z = R.from_euler('z',value-180,degrees=True) + + + def change_scale(self, value): + print(f"Changed scale value to {value}") + #posescale = value/50 + 0.5 + self.posescale = value + + + def change_img_rot(self, val): + print(f"Changed image rotation to {val*90} clockwise") + self.rotate_image = self.img_rot_dict[val] + + + def change_smoothing(self, val): + print(f"Changed smoothing value to {val}") + self.smoothing = val + + + def ready2exit(self): + self.exit_ready = True + + +if __name__ == "__main__": + print("hehe") \ No newline at end of file From 3398acf4c9ff50448a4a1a1ab83c347cce66cb23 Mon Sep 17 00:00:00 2001 From: realdragonturtle <29815169+realdragonturtle@users.noreply.github.com> Date: Thu, 3 Feb 2022 19:30:34 +0100 Subject: [PATCH 04/20] add saving parameters --- bin/guitest.py | 56 ++++++++++++++++--------------------------- bin/inference_gui.py | 23 ++++++++++++++---- bin/mediapipepose.py | 27 +++++++++++---------- bin/parameters.py | 57 ++++++++++++++++++++++++++++++++++++-------- 4 files changed, 101 insertions(+), 62 deletions(-) diff --git a/bin/guitest.py b/bin/guitest.py index 0f848bd..cae8da5 100644 --- a/bin/guitest.py +++ b/bin/guitest.py @@ -74,30 +74,32 @@ def getparams(): hmdwait_check = tk.Checkbutton(text = "Dont wait for hmd", variable = varhmdwait) hmdwait_check.pack() """ - - varclock = tk.IntVar(value = param["rotateclock"]) - rot_clock_check = tk.Checkbutton(text = "Rotate camera clockwise", variable = varclock) - rot_clock_check.pack() + if False: + varclock = tk.IntVar(value = param["rotateclock"]) + rot_clock_check = tk.Checkbutton(text = "Rotate camera clockwise", variable = varclock) + rot_clock_check.pack() - varcclock = tk.IntVar(value = param["rotatecclock"]) - rot_cclock_check = tk.Checkbutton(text = "Rotate camera counter clockwise", variable = varcclock) - rot_cclock_check.pack() + varcclock = tk.IntVar(value = param["rotatecclock"]) + rot_cclock_check = tk.Checkbutton(text = "Rotate camera counter clockwise", variable = varcclock) + rot_cclock_check.pack() varfeet = tk.IntVar(value = param["feetrot"]) rot_feet_check = tk.Checkbutton(text = "Enable experimental foot rotation", variable = varfeet) rot_feet_check.pack() + + if False: - varscale = tk.IntVar(value = param["calib_scale"]) - scale_check = tk.Checkbutton(text = "Enable automatic scale calibration", variable = varscale) - scale_check.pack() - - vartilt = tk.IntVar(value = param["calib_tilt"]) - tilt_check = tk.Checkbutton(text = "Enable automatic tilt calibration", variable = vartilt) - tilt_check.pack() + varscale = tk.IntVar(value = param["calib_scale"]) + scale_check = tk.Checkbutton(text = "Enable automatic scale calibration", variable = varscale) + scale_check.pack() + + vartilt = tk.IntVar(value = param["calib_tilt"]) + tilt_check = tk.Checkbutton(text = "Enable automatic tilt calibration", variable = vartilt) + tilt_check.pack() - varrot = tk.IntVar(value = param["calib_rot"]) - rot_check = tk.Checkbutton(text = "Enable automatic rotation calibration", variable = varrot) - rot_check.pack() + varrot = tk.IntVar(value = param["calib_rot"]) + rot_check = tk.Checkbutton(text = "Enable automatic rotation calibration", variable = varrot) + rot_check.pack() varhip = tk.IntVar(value = param["ignore_hip"]) hip_check = tk.Checkbutton(text = "Don't use hip tracker", variable = varhip) @@ -122,21 +124,10 @@ def getparams(): hmd_to_neck_offset = [float(val) for val in hmdoffsettext.get().split(" ")] preview_skeleton = bool(varskel.get()) dont_wait_hmd = False #bool(varhmdwait.get()) - if varclock.get(): - if varcclock.get(): - rotate_image = cv2.ROTATE_180 - else: - rotate_image = cv2.ROTATE_90_CLOCKWISE - elif varcclock.get(): - rotate_image = cv2.ROTATE_90_COUNTERCLOCKWISE - else: - rotate_image = None + camera_latency = float(camlatencytext.get()) smoothing = float(smoothingtext.get()) feet_rotation = bool(varfeet.get()) - calib_scale = bool(varscale.get()) - calib_tilt = bool(vartilt.get()) - calib_rot = bool(varrot.get()) use_hands = bool(varhand.get()) ignore_hip = bool(varhip.get()) @@ -146,15 +137,10 @@ def getparams(): param["neckoffset"] = hmd_to_neck_offset param["prevskel"] = preview_skeleton param["waithmd"] = dont_wait_hmd - param["rotateclock"] = bool(varclock.get()) - param["rotatecclock"] = bool(varcclock.get()) - param["rotate"] = rotate_image + param["smooth"] = smoothing param["camlatency"] = camera_latency param["feetrot"] = feet_rotation - param["calib_scale"] = calib_scale - param["calib_tilt"] = calib_tilt - param["calib_rot"] = calib_rot param["use_hands"] = use_hands param["ignore_hip"] = ignore_hip diff --git a/bin/inference_gui.py b/bin/inference_gui.py index d1bd6a6..d64197c 100644 --- a/bin/inference_gui.py +++ b/bin/inference_gui.py @@ -3,7 +3,7 @@ import numpy as np from scipy.spatial.transform import Rotation as R import helpers -#import mediapipepose as mep + class InferenceWindow(tk.Frame): def __init__(self, root, params, *args, **kwargs): @@ -24,8 +24,8 @@ def __init__(self, root, params, *args, **kwargs): # calibrate tilt self.calib_tilt_var = tk.BooleanVar(value=self.params.calib_tilt) - self.rot_x_var = tk.DoubleVar(value=self.params.global_rot_x.as_euler('zyx', degrees=True)[2]) - self.rot_z_var = tk.DoubleVar(value=self.params.global_rot_z.as_euler('zyx', degrees=True)[0]) + self.rot_x_var = tk.DoubleVar(value=self.params.global_rot_x.as_euler('zyx', degrees=True)[2]+90) + self.rot_z_var = tk.DoubleVar(value=self.params.global_rot_z.as_euler('zyx', degrees=True)[0]+180) frame2 = tk.Frame(self.root) frame2.pack() @@ -52,6 +52,11 @@ def __init__(self, root, params, *args, **kwargs): frame4.pack() self.change_smooothing_frame(frame4) + # smoothing + frame4_1 = tk.Frame(self.root) + frame4_1.pack() + self.change_cam_lat_frame(frame4_1) + # rotate image frame5 = tk.Frame(self.root) frame5.pack() @@ -169,8 +174,18 @@ def change_smooothing_frame(self, frame): tk.Button(frame, text='Update smoothing value', command=lambda *args: self.params.change_smoothing(float(smoothingtext.get()))).pack(side='left') + def change_cam_lat_frame(self, frame): + + tk.Label(frame, text="Camera latency:", width = 20).pack(side='left') + lat = tk.Entry(frame, width = 10) + lat.pack(side='left') + lat.insert(0, self.params.camera_latency) + + tk.Button(frame, text='Update camera latency', command=lambda *args: self.params.change_camera_latency(float(lat.get()))).pack(side='left') + + def change_image_rotation_frame(self, frame): - rot_img_var = tk.IntVar(value=0)#rotate_image) + rot_img_var = tk.IntVar(value=self.params.img_rot_dict_rev[self.params.rotate_image]) tk.Label(frame, text="Image rotation clockwise:", width = 20).grid(row=0, column=0) tk.Radiobutton(frame, text="0", variable = rot_img_var, value = 0).grid(row=0, column=1) tk.Radiobutton(frame, text="90", variable = rot_img_var, value = 1).grid(row=0, column=2) diff --git a/bin/mediapipepose.py b/bin/mediapipepose.py index 034d9bb..94e57cb 100644 --- a/bin/mediapipepose.py +++ b/bin/mediapipepose.py @@ -7,9 +7,6 @@ sys.path.append(os.getcwd()) #embedable python doesnt find local modules without this line -import tkinter as tk -from tkinter import ttk - import time import threading import cv2 @@ -24,7 +21,7 @@ mp_drawing = mp.solutions.drawing_utils mp_pose = mp.solutions.pose -use_steamvr = True +use_steamvr = False print("Reading parameters...") @@ -126,19 +123,25 @@ def camera_thread_fun(): model_complexity=1) +def shutdown(): + # first save parameters + print("Saving parameters...") + params.save_params() + + cv2.destroyAllWindows() + print("Exiting... You can close the window after 10 seconds.") + exit(0) + cv2.namedWindow("out") #Main program loop: rotation = 0 - i = 0 while(True): # Capture frame-by-frame if params.exit_ready: - cv2.destroyAllWindows() - print("Exiting... You can close the window after 10 seconds.") - exit(0) + shutdown() if params.prev_smoothing != params.smoothing: print(f"Changed smoothing value from {params.prev_smoothing} to {params.smoothing}") @@ -246,9 +249,6 @@ def camera_thread_fun(): #print(f"Inference time: {time.time()-t0}\nSmoothing value: {smoothing}\n") #print how long it took to detect and calculate everything inference_time = time.time() - t0 - - #cv2.imshow("out",image) - #cv2.waitKey(0) img = cv2.cvtColor(img,cv2.COLOR_RGB2BGR) #convert back to bgr and draw the pose mp_drawing.draw_landmarks( @@ -257,8 +257,9 @@ def camera_thread_fun(): cv2.imshow("out",img) #show image, exit program if we press esc if cv2.waitKey(1) == 27: - print("Exiting... You can close the window after 10 seconds.") - exit(0) + #print("Exiting... You can close the window after 10 seconds.") + #exit(0) + shutdown() diff --git a/bin/parameters.py b/bin/parameters.py index ee1ca19..3ed9b33 100644 --- a/bin/parameters.py +++ b/bin/parameters.py @@ -1,7 +1,7 @@ from guitest import getparams from scipy.spatial.transform import Rotation as R import cv2 -import tkinter as tk +import json class Parameters(): def __init__(self) -> None: @@ -15,16 +15,17 @@ def __init__(self) -> None: self.hmd_to_neck_offset = param["neckoffset"] #offset of your hmd to the base of your neck, to ensure the tracking is stable even if you look around. Default is 20cm down, 10cm back. self.preview_skeleton = param["prevskel"] #if True, whole skeleton will appear in vr 2 meters in front of you. Good to visualize if everything is working self.dont_wait_hmd = param["waithmd"] #dont wait for movement from hmd, start inference immediately. - self.rotate_image = param["rotate"] # cv2.ROTATE_90_CLOCKWISE # cv2.ROTATE_90_COUTERCLOCKWISE # cv2.ROTATE_180 # None # if you want, rotate the camera + self.rotate_image = 0 # cv2.ROTATE_90_CLOCKWISE # cv2.ROTATE_90_COUTERCLOCKWISE # cv2.ROTATE_180 # None # if you want, rotate the camera self.camera_latency = param["camlatency"] self.smoothing = param["smooth"] self.feet_rotation = param["feetrot"] - self.calib_scale = param["calib_scale"] - self.calib_tilt = param["calib_tilt"] - self.calib_rot = param["calib_rot"] self.use_hands = param["use_hands"] self.ignore_hip = param["ignore_hip"] + self.calib_rot = True + self.calib_tilt = True + self.calib_scale = True + self.prev_smoothing = self.smoothing self.recalibrate = False @@ -38,12 +39,9 @@ def __init__(self) -> None: self.exit_ready = False self.img_rot_dict = {0: None, 1: cv2.ROTATE_90_CLOCKWISE, 2: cv2.ROTATE_180, 3: cv2.ROTATE_90_COUNTERCLOCKWISE} + self.img_rot_dict_rev = {None: 0, cv2.ROTATE_90_CLOCKWISE: 1, cv2.ROTATE_180: 2, cv2.ROTATE_90_COUNTERCLOCKWISE: 3} - # trace variables - - #self.root = tk.Tk() - #self.rot_y_var = tk.DoubleVar(value=self.global_rot_y.as_euler('zyx', degrees=True)[1]) - + self.load_params() def change_recalibrate(self): @@ -80,9 +78,48 @@ def change_smoothing(self, val): self.smoothing = val + def change_camera_latency(self, val): + print(f"Changed camera latency to {val}") + self.camera_latency = val + + def ready2exit(self): self.exit_ready = True + def save_params(self): + param = {} + param["rotate"] = self.img_rot_dict_rev[self.rotate_image] + param["smooth"] = self.smoothing + + param["camlatency"] = self.camera_latency + + param["roty"] = self.global_rot_y.as_euler('zyx', degrees=True)[1] + param["rotx"] = self.global_rot_x.as_euler('zyx', degrees=True)[2] + param["rotz"] = self.global_rot_z.as_euler('zyx', degrees=True)[0] + param["scale"] = self.posescale + + with open("saved_params.json", "w") as f: + json.dump(param, f) + + def load_params(self): + + try: + with open("saved_params.json", "r") as f: + param = json.load(f) + + self.rotate_image = self.img_rot_dict[param["rotate"]] + self.smoothing = param["smooth"] + self.camera_latency = param["camlatency"] + + self.global_rot_y = R.from_euler('y',param["roty"],degrees=True) + self.global_rot_x = R.from_euler('x',param["rotx"],degrees=True) + self.global_rot_z = R.from_euler('z',param["rotz"],degrees=True) + self.posescale = param["scale"] + except: + print("Save file not found, will be created after you exit the program.") + + + if __name__ == "__main__": print("hehe") \ No newline at end of file From 89d55c705f00a4df54669e636836c728ecdead10 Mon Sep 17 00:00:00 2001 From: ju1ce Date: Sun, 20 Feb 2022 14:20:57 +0100 Subject: [PATCH 05/20] added more camera settings. added additional smoothing. added toggles to disable smoothing options. added a button to pause tracking. --- bin/.gitignore | 4 +- bin/inference_gui.py | 65 ++++-- bin/{guitest.py => init_gui.py} | 338 +++++++++++++++++--------------- bin/mediapipepose.py | 44 ++++- bin/parameters.py | 22 ++- bin/params.p | Bin 300 -> 0 bytes 6 files changed, 286 insertions(+), 187 deletions(-) rename bin/{guitest.py => init_gui.py} (72%) delete mode 100644 bin/params.p diff --git a/bin/.gitignore b/bin/.gitignore index e6cfaef..4213963 100644 --- a/bin/.gitignore +++ b/bin/.gitignore @@ -1,3 +1,5 @@ __pycache__ movenet_model* -checkpoint* \ No newline at end of file +checkpoint* +saved_params.json +params.p \ No newline at end of file diff --git a/bin/inference_gui.py b/bin/inference_gui.py index d64197c..0d33e33 100644 --- a/bin/inference_gui.py +++ b/bin/inference_gui.py @@ -4,6 +4,7 @@ from scipy.spatial.transform import Rotation as R import helpers +use_steamvr = False class InferenceWindow(tk.Frame): def __init__(self, root, params, *args, **kwargs): @@ -46,12 +47,21 @@ def __init__(self, root, params, *args, **kwargs): # recalibrate tk.Button(self.root, text='Recalibrate (automatically recalibrates checked values above)', command=self.autocalibrate).pack() + + # pause tracking + tk.Button(self.root, text='Pause/Unpause tracking', + command=self.pause_tracking).pack() # smoothing frame4 = tk.Frame(self.root) frame4.pack() self.change_smooothing_frame(frame4) + # smoothing + frame4_2 = tk.Frame(self.root) + frame4_2.pack() + self.change_add_smoothing_frame(frame4_2) + # smoothing frame4_1 = tk.Frame(self.root) frame4_1.pack() @@ -76,11 +86,11 @@ def set_rot_y_var(self): def set_rot_z_var(self): - self.rot_z_var.set(self.params.global_rot_z.as_euler('zyx', degrees=True)[0]) + self.rot_z_var.set(self.params.global_rot_z.as_euler('zyx', degrees=True)[0]+180) def set_rot_x_var(self): - self.rot_x_var.set(self.params.global_rot_x.as_euler('zyx', degrees=True)[2]) + self.rot_x_var.set(self.params.global_rot_x.as_euler('zyx', degrees=True)[2]+90) #self.root.after(0, self.set_rot_x_var) @@ -166,12 +176,13 @@ def calibrate_scale_frame(self, frame): def change_smooothing_frame(self, frame): - tk.Label(frame, text="Smoothing:", width = 20).pack(side='left') + tk.Label(frame, text="Smoothing window:", width = 20).pack(side='left') smoothingtext = tk.Entry(frame, width = 10) smoothingtext.pack(side='left') smoothingtext.insert(0, self.params.smoothing) - tk.Button(frame, text='Update smoothing value', command=lambda *args: self.params.change_smoothing(float(smoothingtext.get()))).pack(side='left') + tk.Button(frame, text='Update', command=lambda *args: self.params.change_smoothing(float(smoothingtext.get()))).pack(side='left') + tk.Button(frame, text='Disable', command=lambda *args: self.params.change_smoothing(0.0)).pack(side='left') def change_cam_lat_frame(self, frame): @@ -181,7 +192,17 @@ def change_cam_lat_frame(self, frame): lat.pack(side='left') lat.insert(0, self.params.camera_latency) - tk.Button(frame, text='Update camera latency', command=lambda *args: self.params.change_camera_latency(float(lat.get()))).pack(side='left') + tk.Button(frame, text='Update', command=lambda *args: self.params.change_camera_latency(float(lat.get()))).pack(side='left') + + def change_add_smoothing_frame(self, frame): + + tk.Label(frame, text="Additional smoothing:", width = 20).pack(side='left') + lat = tk.Entry(frame, width = 10) + lat.pack(side='left') + lat.insert(0, self.params.additional_smoothing) + + tk.Button(frame, text='Update', command=lambda *args: self.params.change_additional_smoothing(float(lat.get()))).pack(side='left') + tk.Button(frame, text='Disable', command=lambda *args: self.params.change_additional_smoothing(0.0)).pack(side='left') def change_image_rotation_frame(self, frame): @@ -197,22 +218,23 @@ def change_image_rotation_frame(self, frame): def autocalibrate(self): - for _ in range(10): - array = helpers.sendToSteamVR("getdevicepose 0") #get hmd data to allign our skeleton to + if use_steamvr: + for _ in range(10): + array = helpers.sendToSteamVR("getdevicepose 0") #get hmd data to allign our skeleton to - if "error" in array: #continue to next iteration if there is an error - continue - else: - break + if "error" in array: #continue to next iteration if there is an error + continue + else: + break - if "error" in array: #continue to next iteration if there is an error - print("Failed to contact SteamVR after 10 tries... Try to autocalibrate again.") - return + if "error" in array: #continue to next iteration if there is an error + print("Failed to contact SteamVR after 10 tries... Try to autocalibrate again.") + return - headsetpos = [float(array[3]),float(array[4]),float(array[5])] - headsetrot = R.from_quat([float(array[7]),float(array[8]),float(array[9]),float(array[6])]) - - neckoffset = headsetrot.apply(self.params.hmd_to_neck_offset) #the neck position seems to be the best point to allign to, as its well defined on + headsetpos = [float(array[3]),float(array[4]),float(array[5])] + headsetrot = R.from_quat([float(array[7]),float(array[8]),float(array[9]),float(array[6])]) + + neckoffset = headsetrot.apply(self.params.hmd_to_neck_offset) #the neck position seems to be the best point to allign to, as its well defined on #the skeleton (unlike the eyes/nose, which jump around) and can be calculated from hmd. if self.params.calib_tilt: @@ -254,7 +276,7 @@ def autocalibrate(self): print("Postcalib x angle: ", value * 57.295779513) - if self.params.calib_rot: + if use_steamvr and self.params.calib_rot: feet_rot = self.params.pose3d_og[0] - self.params.pose3d_og[5] value = np.arctan2(feet_rot[0],feet_rot[2]) value_hmd = np.arctan2(headsetrot.as_matrix()[0][0],headsetrot.as_matrix()[2][0]) @@ -274,7 +296,7 @@ def autocalibrate(self): print("Postcalib y value: ", value * 57.295779513) - if self.params.calib_scale: + if use_steamvr and self.params.calib_scale: #calculate the height of the skeleton, calculate the height in steamvr as distance of hmd from the ground. #divide the two to get scale skelSize = np.max(self.params.pose3d_og, axis=0)-np.min(self.params.pose3d_og, axis=0) @@ -288,6 +310,9 @@ def autocalibrate(self): def put_separator(self): separator = ttk.Separator(self.root, orient='horizontal') separator.pack(fill='x') + + def pause_tracking(self): + self.params.paused = not self.params.paused def make_inference_gui(_params): diff --git a/bin/guitest.py b/bin/init_gui.py similarity index 72% rename from bin/guitest.py rename to bin/init_gui.py index cae8da5..3880266 100644 --- a/bin/guitest.py +++ b/bin/init_gui.py @@ -1,154 +1,184 @@ -import tkinter as tk -import pickle -import cv2 - -def getparams(): - - try: - param = pickle.load(open("params.p","rb")) - except: - param = {} - - if "camid" not in param: - param["camid"] = 'http://192.168.1.103:8080/video' - if "imgsize" not in param: - param["imgsize"] = 800 - if "neckoffset" not in param: - param["neckoffset"] = [0.0, -0.2, 0.1] - if "prevskel" not in param: - param["prevskel"] = False - if "waithmd" not in param: - param["waithmd"] = False - if "rotateclock" not in param: - param["rotateclock"] = False - if "rotatecclock" not in param: - param["rotatecclock"] = False - if "rotate" not in param: - param["rotate"] = None - if "camlatency" not in param: - param["camlatency"] = 0.05 - if "smooth" not in param: - param["smooth"] = 0.5 - if "feetrot" not in param: - param["feetrot"] = False - if "calib_scale" not in param: - param["calib_scale"] = True - if "calib_tilt" not in param: - param["calib_tilt"] = True - if "calib_rot" not in param: - param["calib_rot"] = True - if "use_hands" not in param: - param["use_hands"] = False - if "ignore_hip" not in param: - param["ignore_hip"] = False - - window = tk.Tk() - - tk.Label(text="Camera IP or ID:", width = 50).pack() - camid = tk.Entry(width = 50) - camid.pack() - camid.insert(0,param["camid"]) - - tk.Label(text="Maximum image size:", width = 50).pack() - maximgsize = tk.Entry(width = 20) - maximgsize.pack() - maximgsize.insert(0,param["imgsize"]) - - tk.Label(text="Offset of HMD to neck:", width = 50).pack() - hmdoffsettext = tk.Entry(width = 50) - hmdoffsettext.pack() - hmdoffsettext.insert(0," ".join(map(str,param["neckoffset"]))) - - tk.Label(text="Smoothing:", width = 50).pack() - smoothingtext = tk.Entry(width = 50) - smoothingtext.pack() - smoothingtext.insert(0,param["smooth"]) - - tk.Label(text="Camera latency:", width = 50).pack() - camlatencytext = tk.Entry(width = 50) - camlatencytext.pack() - camlatencytext.insert(0,param["camlatency"]) - - """ - varhmdwait = tk.IntVar(value = param["waithmd"]) - hmdwait_check = tk.Checkbutton(text = "Dont wait for hmd", variable = varhmdwait) - hmdwait_check.pack() - """ - if False: - varclock = tk.IntVar(value = param["rotateclock"]) - rot_clock_check = tk.Checkbutton(text = "Rotate camera clockwise", variable = varclock) - rot_clock_check.pack() - - varcclock = tk.IntVar(value = param["rotatecclock"]) - rot_cclock_check = tk.Checkbutton(text = "Rotate camera counter clockwise", variable = varcclock) - rot_cclock_check.pack() - - varfeet = tk.IntVar(value = param["feetrot"]) - rot_feet_check = tk.Checkbutton(text = "Enable experimental foot rotation", variable = varfeet) - rot_feet_check.pack() - - if False: - - varscale = tk.IntVar(value = param["calib_scale"]) - scale_check = tk.Checkbutton(text = "Enable automatic scale calibration", variable = varscale) - scale_check.pack() - - vartilt = tk.IntVar(value = param["calib_tilt"]) - tilt_check = tk.Checkbutton(text = "Enable automatic tilt calibration", variable = vartilt) - tilt_check.pack() - - varrot = tk.IntVar(value = param["calib_rot"]) - rot_check = tk.Checkbutton(text = "Enable automatic rotation calibration", variable = varrot) - rot_check.pack() - - varhip = tk.IntVar(value = param["ignore_hip"]) - hip_check = tk.Checkbutton(text = "Don't use hip tracker", variable = varhip) - hip_check.pack() - - tk.Label(text="-"*50, width = 50).pack() - - varhand = tk.IntVar(value = param["use_hands"]) - hand_check = tk.Checkbutton(text = "DEV: spawn trackers for hands", variable = varhand) - hand_check.pack() - - varskel = tk.IntVar(value = param["prevskel"]) - skeleton_check = tk.Checkbutton(text = "DEV: preview skeleton in VR", variable = varskel) - skeleton_check.pack() - - tk.Button(text='Save and continue', command=window.quit).pack() - - window.mainloop() - - cameraid = camid.get() - maximgsize = int(maximgsize.get()) - hmd_to_neck_offset = [float(val) for val in hmdoffsettext.get().split(" ")] - preview_skeleton = bool(varskel.get()) - dont_wait_hmd = False #bool(varhmdwait.get()) - - camera_latency = float(camlatencytext.get()) - smoothing = float(smoothingtext.get()) - feet_rotation = bool(varfeet.get()) - use_hands = bool(varhand.get()) - ignore_hip = bool(varhip.get()) - - param = {} - param["camid"] = cameraid - param["imgsize"] = maximgsize - param["neckoffset"] = hmd_to_neck_offset - param["prevskel"] = preview_skeleton - param["waithmd"] = dont_wait_hmd - - param["smooth"] = smoothing - param["camlatency"] = camera_latency - param["feetrot"] = feet_rotation - param["use_hands"] = use_hands - param["ignore_hip"] = ignore_hip - - pickle.dump(param,open("params.p","wb")) - - window.destroy() - - return param - -if __name__ == "__main__": - print(getparams()) +import tkinter as tk +import pickle +import cv2 + +def getparams(): + + try: + param = pickle.load(open("params.p","rb")) + except: + param = {} + + if "camid" not in param: + param["camid"] = 'http://192.168.1.103:8080/video' + if "imgsize" not in param: + param["imgsize"] = 800 + if "neckoffset" not in param: + param["neckoffset"] = [0.0, -0.2, 0.1] + if "prevskel" not in param: + param["prevskel"] = False + if "waithmd" not in param: + param["waithmd"] = False + if "rotateclock" not in param: + param["rotateclock"] = False + if "rotatecclock" not in param: + param["rotatecclock"] = False + if "rotate" not in param: + param["rotate"] = None + if "camlatency" not in param: + param["camlatency"] = 0.05 + if "smooth" not in param: + param["smooth"] = 0.5 + if "feetrot" not in param: + param["feetrot"] = False + if "calib_scale" not in param: + param["calib_scale"] = True + if "calib_tilt" not in param: + param["calib_tilt"] = True + if "calib_rot" not in param: + param["calib_rot"] = True + if "use_hands" not in param: + param["use_hands"] = False + if "ignore_hip" not in param: + param["ignore_hip"] = False + if "camera_settings" not in param: + param["camera_settings"] = False + if "camera_width" not in param: + param["camera_width"] = 640 + if "camera_height" not in param: + param["camera_height"] = 480 + + window = tk.Tk() + + tk.Label(text="Camera IP or ID:", width = 50).pack() + camid = tk.Entry(width = 50) + camid.pack() + camid.insert(0,param["camid"]) + + tk.Label(text="NOTE: Increasing resolution may decrease performance.\n Unless you have problems with opening camera, leave it at default.", width = 50).pack() + + tk.Label(text="Camera width:", width = 50).pack() + camwidth = tk.Entry(width = 20) + camwidth.pack() + camwidth.insert(0,param["camera_width"]) + + tk.Label(text="Camera height:", width = 50).pack() + camheight = tk.Entry(width = 20) + camheight.pack() + camheight.insert(0,param["camera_height"]) + + varcamsettings = tk.IntVar(value = param["camera_settings"]) + cam_settings_check = tk.Checkbutton(text = "Attempt to open camera settings", variable = varcamsettings) + cam_settings_check.pack() + + tk.Label(text="Maximum image size:", width = 50).pack() + maximgsize = tk.Entry(width = 20) + maximgsize.pack() + maximgsize.insert(0,param["imgsize"]) + + tk.Label(text="Offset of HMD to neck:", width = 50).pack() + hmdoffsettext = tk.Entry(width = 50) + hmdoffsettext.pack() + hmdoffsettext.insert(0," ".join(map(str,param["neckoffset"]))) + + if False: + tk.Label(text="Smoothing:", width = 50).pack() + smoothingtext = tk.Entry(width = 50) + smoothingtext.pack() + smoothingtext.insert(0,param["smooth"]) + + tk.Label(text="Camera latency:", width = 50).pack() + camlatencytext = tk.Entry(width = 50) + camlatencytext.pack() + camlatencytext.insert(0,param["camlatency"]) + + """ + varhmdwait = tk.IntVar(value = param["waithmd"]) + hmdwait_check = tk.Checkbutton(text = "Dont wait for hmd", variable = varhmdwait) + hmdwait_check.pack() + """ + if False: + varclock = tk.IntVar(value = param["rotateclock"]) + rot_clock_check = tk.Checkbutton(text = "Rotate camera clockwise", variable = varclock) + rot_clock_check.pack() + + varcclock = tk.IntVar(value = param["rotatecclock"]) + rot_cclock_check = tk.Checkbutton(text = "Rotate camera counter clockwise", variable = varcclock) + rot_cclock_check.pack() + + varfeet = tk.IntVar(value = param["feetrot"]) + rot_feet_check = tk.Checkbutton(text = "Enable experimental foot rotation", variable = varfeet) + rot_feet_check.pack() + + if False: + + varscale = tk.IntVar(value = param["calib_scale"]) + scale_check = tk.Checkbutton(text = "Enable automatic scale calibration", variable = varscale) + scale_check.pack() + + vartilt = tk.IntVar(value = param["calib_tilt"]) + tilt_check = tk.Checkbutton(text = "Enable automatic tilt calibration", variable = vartilt) + tilt_check.pack() + + varrot = tk.IntVar(value = param["calib_rot"]) + rot_check = tk.Checkbutton(text = "Enable automatic rotation calibration", variable = varrot) + rot_check.pack() + + varhip = tk.IntVar(value = param["ignore_hip"]) + hip_check = tk.Checkbutton(text = "Don't use hip tracker", variable = varhip) + hip_check.pack() + + tk.Label(text="-"*50, width = 50).pack() + + varhand = tk.IntVar(value = param["use_hands"]) + hand_check = tk.Checkbutton(text = "DEV: spawn trackers for hands", variable = varhand) + hand_check.pack() + + varskel = tk.IntVar(value = param["prevskel"]) + skeleton_check = tk.Checkbutton(text = "DEV: preview skeleton in VR", variable = varskel) + skeleton_check.pack() + + tk.Button(text='Save and continue', command=window.quit).pack() + + window.mainloop() + + cameraid = camid.get() + maximgsize = int(maximgsize.get()) + hmd_to_neck_offset = [float(val) for val in hmdoffsettext.get().split(" ")] + preview_skeleton = bool(varskel.get()) + dont_wait_hmd = False #bool(varhmdwait.get()) + + #camera_latency = float(camlatencytext.get()) + #smoothing = float(smoothingtext.get()) + feet_rotation = bool(varfeet.get()) + use_hands = bool(varhand.get()) + ignore_hip = bool(varhip.get()) + camera_settings = bool(varcamsettings.get()) + camera_height = camheight.get() + camera_width = camwidth.get() + + param = {} + param["camid"] = cameraid + param["imgsize"] = maximgsize + param["neckoffset"] = hmd_to_neck_offset + param["prevskel"] = preview_skeleton + param["waithmd"] = dont_wait_hmd + + #param["smooth"] = smoothing + #param["camlatency"] = camera_latency + param["feetrot"] = feet_rotation + param["use_hands"] = use_hands + param["ignore_hip"] = ignore_hip + + param["camera_settings"] = camera_settings + param["camera_height"] = camera_height + param["camera_width"] = camera_width + + pickle.dump(param,open("params.p","wb")) + + window.destroy() + + return param + +if __name__ == "__main__": + print(getparams()) diff --git a/bin/mediapipepose.py b/bin/mediapipepose.py index 94e57cb..4d86112 100644 --- a/bin/mediapipepose.py +++ b/bin/mediapipepose.py @@ -42,7 +42,23 @@ def camera_thread_fun(): else: cameraid = params.cameraid - cap = cv2.VideoCapture(cameraid) + if params.camera_settings: + cap = cv2.VideoCapture(cameraid, 700) + cap.set(cv2.CAP_PROP_SETTINGS,1) + else: + cap = cv2.VideoCapture(cameraid) + + #codec = 0x47504A4D + #cap.set(cv2.CAP_PROP_FOURCC, codec) + + print( params.camera_height) + + if params.camera_height != 0: + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, int(params.camera_height)) + + if params.camera_width != 0: + cap.set(cv2.CAP_PROP_FRAME_WIDTH, int(params.camera_width)) + print("Camera opened!") while True: ret, image_from_thread = cap.read() @@ -108,9 +124,9 @@ def camera_thread_fun(): time.sleep(0.2) time.sleep(0.2) - resp = sendToSteamVR(f"settings 50 {params.smoothing}") + resp = sendToSteamVR(f"settings 50 {params.smoothing} {params.additional_smoothing}") while "error" in resp: - resp = sendToSteamVR(f"settings 50 {params.smoothing}") + resp = sendToSteamVR(f"settings 50 {params.smoothing} {params.additional_smoothing}") print(resp) time.sleep(1) @@ -138,19 +154,26 @@ def shutdown(): rotation = 0 i = 0 + +prev_smoothing = params.smoothing +prev_add_smoothing = params.additional_smoothing + while(True): # Capture frame-by-frame if params.exit_ready: shutdown() - if params.prev_smoothing != params.smoothing: - print(f"Changed smoothing value from {params.prev_smoothing} to {params.smoothing}") - params.prev_smoothing = params.smoothing + if prev_smoothing != params.smoothing or prev_add_smoothing != params.additional_smoothing: + print(f"Changed smoothing value from {prev_smoothing} to {params.smoothing}") + print(f"Changed additional smoothing from {prev_add_smoothing} to {params.additional_smoothing}") + + prev_smoothing = params.smoothing + prev_add_smoothing = params.additional_smoothing if use_steamvr: - resp = sendToSteamVR(f"settings 50 {params.smoothing}") + resp = sendToSteamVR(f"settings 50 {params.smoothing} {params.additional_smoothing}") while "error" in resp: - resp = sendToSteamVR(f"settings 50 {params.smoothing}") + resp = sendToSteamVR(f"settings 50 {params.smoothing} {params.additional_smoothing}") print(resp) time.sleep(1) @@ -168,6 +191,11 @@ def shutdown(): ratio = max(img.shape)/params.maximgsize img = cv2.resize(img,(int(img.shape[1]/ratio),int(img.shape[0]/ratio))) + if params.paused: + cv2.imshow("out",img) + cv2.waitKey(1) + continue + img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB) #print(image.shape) diff --git a/bin/parameters.py b/bin/parameters.py index 3ed9b33..71511af 100644 --- a/bin/parameters.py +++ b/bin/parameters.py @@ -1,4 +1,4 @@ -from guitest import getparams +from init_gui import getparams from scipy.spatial.transform import Rotation as R import cv2 import json @@ -16,11 +16,18 @@ def __init__(self) -> None: self.preview_skeleton = param["prevskel"] #if True, whole skeleton will appear in vr 2 meters in front of you. Good to visualize if everything is working self.dont_wait_hmd = param["waithmd"] #dont wait for movement from hmd, start inference immediately. self.rotate_image = 0 # cv2.ROTATE_90_CLOCKWISE # cv2.ROTATE_90_COUTERCLOCKWISE # cv2.ROTATE_180 # None # if you want, rotate the camera - self.camera_latency = param["camlatency"] - self.smoothing = param["smooth"] + #self.camera_latency = param["camlatency"] + #self.smoothing = param["smooth"] + self.camera_latency = 0.1 + self.smoothing = 0.5 + self.additional_smoothing = 0 self.feet_rotation = param["feetrot"] self.use_hands = param["use_hands"] self.ignore_hip = param["ignore_hip"] + + self.camera_settings = param["camera_settings"] + self.camera_width = param["camera_width"] + self.camera_height = param["camera_height"] self.calib_rot = True self.calib_tilt = True @@ -41,6 +48,8 @@ def __init__(self) -> None: self.img_rot_dict = {0: None, 1: cv2.ROTATE_90_CLOCKWISE, 2: cv2.ROTATE_180, 3: cv2.ROTATE_90_COUNTERCLOCKWISE} self.img_rot_dict_rev = {None: 0, cv2.ROTATE_90_CLOCKWISE: 1, cv2.ROTATE_180: 2, cv2.ROTATE_90_COUNTERCLOCKWISE: 3} + self.paused = False + self.load_params() @@ -76,7 +85,10 @@ def change_img_rot(self, val): def change_smoothing(self, val): print(f"Changed smoothing value to {val}") self.smoothing = val - + + def change_additional_smoothing(self, val): + print(f"Changed additional smoothing value to {val}") + self.additional_smoothing = val def change_camera_latency(self, val): print(f"Changed camera latency to {val}") @@ -93,6 +105,7 @@ def save_params(self): param["smooth"] = self.smoothing param["camlatency"] = self.camera_latency + param["addsmooth"] = self.additional_smoothing param["roty"] = self.global_rot_y.as_euler('zyx', degrees=True)[1] param["rotx"] = self.global_rot_x.as_euler('zyx', degrees=True)[2] @@ -111,6 +124,7 @@ def load_params(self): self.rotate_image = self.img_rot_dict[param["rotate"]] self.smoothing = param["smooth"] self.camera_latency = param["camlatency"] + self.additional_smoothing = param["addsmooth"] self.global_rot_y = R.from_euler('y',param["roty"],degrees=True) self.global_rot_x = R.from_euler('x',param["rotx"],degrees=True) diff --git a/bin/params.p b/bin/params.p deleted file mode 100644 index facec7b2fdecbcbc226e4de069fa074d0fbf14db..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 300 zcmXYrzfQwI490_ircHSVVC$4WAfeEuV+O<{pekIP=9Eh=%3X^BP)i z@F+OFda-}DKl}SQ`@BB!4)IPiJQsN*3&P9F+HM|2$^A{3+@)a>Ch_e&jng>V$O1H> zFPoK-Z$OV%rx>&#YF(BFEIrXIa|b;y`}zGE;yYXHT>&`mdf1p66z$M|$)&BDg3D>I zZEis*Eo$20#StCvaO9XCJRF%uYg+m1cOmMjX From 6d74eb30eadc7848a2495077e6926e55af2f21ef Mon Sep 17 00:00:00 2001 From: ju1ce Date: Sun, 20 Feb 2022 14:30:48 +0100 Subject: [PATCH 06/20] switched legs as they seemed wrong when foot rotation is enabled --- bin/helpers.py | 2 +- bin/mediapipepose.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/bin/helpers.py b/bin/helpers.py index 065d51f..c683b19 100644 --- a/bin/helpers.py +++ b/bin/helpers.py @@ -225,7 +225,7 @@ def get_rot_mediapipe(pose3d): r_foot_rot = R.from_matrix(r_foot_rot).as_quat() l_foot_rot = R.from_matrix(l_foot_rot).as_quat() - return hip_rot, l_foot_rot, r_foot_rot + return hip_rot, r_foot_rot, l_foot_rot diff --git a/bin/mediapipepose.py b/bin/mediapipepose.py index 4d86112..cca8b12 100644 --- a/bin/mediapipepose.py +++ b/bin/mediapipepose.py @@ -51,8 +51,6 @@ def camera_thread_fun(): #codec = 0x47504A4D #cap.set(cv2.CAP_PROP_FOURCC, codec) - print( params.camera_height) - if params.camera_height != 0: cap.set(cv2.CAP_PROP_FRAME_HEIGHT, int(params.camera_height)) From 56e25989b4dea801b39f71a33d62b7e929b66deb Mon Sep 17 00:00:00 2001 From: ju1ce Date: Sat, 26 Feb 2022 13:26:40 +0100 Subject: [PATCH 07/20] fixed rotation calibration values --- bin/inference_gui.py | 28 ++++++++++++++++++++++------ bin/mediapipepose.py | 2 +- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/bin/inference_gui.py b/bin/inference_gui.py index 0d33e33..0f7e00c 100644 --- a/bin/inference_gui.py +++ b/bin/inference_gui.py @@ -4,7 +4,7 @@ from scipy.spatial.transform import Rotation as R import helpers -use_steamvr = False +use_steamvr = True class InferenceWindow(tk.Frame): def __init__(self, root, params, *args, **kwargs): @@ -81,7 +81,14 @@ def __init__(self, root, params, *args, **kwargs): def set_rot_y_var(self): - self.rot_y_var.set(self.params.global_rot_y.as_euler('zyx', degrees=True)[1]) + angle = -(180+self.params.global_rot_y.as_euler('zyx', degrees=True)[1]) + #print("calculated angle from rot is:",angle) + if angle >= 360: + angle -= 360 + elif angle < 0: + angle += 360 + #print("calculated angle final is:",angle) + self.rot_y_var.set(angle) #self.root.after(0, self.set_rot_y_var) @@ -135,7 +142,7 @@ def calibrate_tilt_frame(self, frame): rot_z_frame = tk.Frame(frame) rot_z_frame.pack() - rot_x = tk.Scale(rot_x_frame, label="Roation x:", from_=90, to=180, #command=lambda *args: self.params.rot_change_x(self.rot_x_var.get()), + rot_x = tk.Scale(rot_x_frame, label="Roation x:", from_=0, to=180, #command=lambda *args: self.params.rot_change_x(self.rot_x_var.get()), orient=tk.HORIZONTAL, length=400, showvalue=1, tickinterval=15, variable=self.rot_x_var) rot_x.pack(expand=True,fill='both',side='left') self.rot_x_var.trace_add('write', callback=lambda var, index, mode: self.params.rot_change_x(self.rot_x_var.get())) @@ -143,7 +150,7 @@ def calibrate_tilt_frame(self, frame): tk.Button(rot_x_frame, text="<", command=lambda *args: self.rot_x_var.set(self.rot_x_var.get()-1), width=10).pack(expand=True,fill='both',side='left') tk.Button(rot_x_frame, text=">", command=lambda *args: self.rot_x_var.set(self.rot_x_var.get()+1), width=10).pack(expand=True,fill='both',side='left') - rot_z = tk.Scale(rot_z_frame, label="Roation z:", from_=180, to=360, #command=lambda *args: self.params.rot_change_z(self.rot_z_var.get()), + rot_z = tk.Scale(rot_z_frame, label="Roation z:", from_=90, to=270, #command=lambda *args: self.params.rot_change_z(self.rot_z_var.get()), orient=tk.HORIZONTAL, length=400, showvalue=1, tickinterval=30, variable=self.rot_z_var) rot_z.pack(expand=True,fill='both',side='left') self.rot_z_var.trace_add('write', callback=lambda var, index, mode: self.params.rot_change_z(self.rot_z_var.get())) @@ -281,11 +288,20 @@ def autocalibrate(self): value = np.arctan2(feet_rot[0],feet_rot[2]) value_hmd = np.arctan2(headsetrot.as_matrix()[0][0],headsetrot.as_matrix()[2][0]) print("Precalib y value: ", value * 57.295779513) - print("hmd y value: ", value_hmd * 57.295779513) + print("hmd y value: ", value_hmd * 57.295779513) value = value - value_hmd - self.params.global_rot_y = R.from_euler('y',-value) + value = -value + + print("Calibrate to value:", value * 57.295779513) + + self.params.global_rot_y = R.from_euler('y',value) + + + angle = self.params.global_rot_y.as_euler('zyx', degrees=True) + print("angle from rot = ", -(180+angle[1])) + self.set_rot_y_var() for j in range(self.params.pose3d_og.shape[0]): diff --git a/bin/mediapipepose.py b/bin/mediapipepose.py index cca8b12..f9f2aad 100644 --- a/bin/mediapipepose.py +++ b/bin/mediapipepose.py @@ -21,7 +21,7 @@ mp_drawing = mp.solutions.drawing_utils mp_pose = mp.solutions.pose -use_steamvr = False +use_steamvr = True print("Reading parameters...") From 2beb45bcca68f59c56ef8dd00a36c5b07e1e403a Mon Sep 17 00:00:00 2001 From: ju1ce Date: Mon, 28 Feb 2022 16:02:48 +0100 Subject: [PATCH 08/20] Added option to flip rotation calibration. Added FPS counter --- bin/inference_gui.py | 15 +++++++++++++-- bin/mediapipepose.py | 2 +- bin/parameters.py | 14 ++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/bin/inference_gui.py b/bin/inference_gui.py index 0f7e00c..24bc6b6 100644 --- a/bin/inference_gui.py +++ b/bin/inference_gui.py @@ -15,6 +15,7 @@ def __init__(self, root, params, *args, **kwargs): # calibrate rotation self.calib_rot_var = tk.BooleanVar(value=self.params.calib_rot) + self.calib_flip_var = tk.BooleanVar(value=self.params.flip) self.rot_y_var = tk.DoubleVar(value=self.params.global_rot_y.as_euler('zyx', degrees=True)[1]) frame1 = tk.Frame(self.root) @@ -82,6 +83,10 @@ def __init__(self, root, params, *args, **kwargs): def set_rot_y_var(self): angle = -(180+self.params.global_rot_y.as_euler('zyx', degrees=True)[1]) + + if not self.params.flip: + angle += 180 + #print("calculated angle from rot is:",angle) if angle >= 360: angle -= 360 @@ -108,16 +113,22 @@ def set_scale_var(self): def change_rot_auto(self): self.params.calib_rot = self.calib_rot_var.get() print(f"Mark rotation to{'' if self.params.calib_rot else ' NOT'} be automatically calibrated") + + def change_rot_flip(self): + self.params.flip = self.calib_flip_var.get() + print("changed flip to: ", self.params.flip) def calibrate_rotation_frame(self, frame): rot_check = tk.Checkbutton(frame, text = "Enable automatic rotation calibration", variable = self.calib_rot_var, command=self.change_rot_auto)#, command=lambda *args: show_hide(varrot, [rot_y_frame])) + flip_check = tk.Checkbutton(frame, text = "Flip calibration", variable = self.calib_flip_var, command=self.change_rot_flip) rot_y_frame = tk.Frame(frame) rot_check.pack() + flip_check.pack() rot_y_frame.pack() - rot_y = tk.Scale(rot_y_frame, label="Roation y:", from_=0, to=360, #command=lambda *args: self.params.rot_change_y(self.rot_y_var.get()), + rot_y = tk.Scale(rot_y_frame, label="Roation y:", from_=-40, to=400, #command=lambda *args: self.params.rot_change_y(self.rot_y_var.get()), orient=tk.HORIZONTAL, length=400, showvalue=1, tickinterval=60, variable=self.rot_y_var) rot_y.pack(expand=True,fill='both',side='left') @@ -300,7 +311,7 @@ def autocalibrate(self): angle = self.params.global_rot_y.as_euler('zyx', degrees=True) - print("angle from rot = ", -(180+angle[1])) + print("angle from rot = ", -(180+angle[1]), "whole:",angle) self.set_rot_y_var() diff --git a/bin/mediapipepose.py b/bin/mediapipepose.py index f9f2aad..cda569b 100644 --- a/bin/mediapipepose.py +++ b/bin/mediapipepose.py @@ -279,7 +279,7 @@ def shutdown(): img = cv2.cvtColor(img,cv2.COLOR_RGB2BGR) #convert back to bgr and draw the pose mp_drawing.draw_landmarks( img, results.pose_landmarks, mp_pose.POSE_CONNECTIONS) - img = cv2.putText(img, f"{inference_time:1.3f}", (10,30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 1, cv2.LINE_AA) + img = cv2.putText(img, f"{inference_time:1.3f}, FPS:{int(1/inference_time)}", (10,30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 1, cv2.LINE_AA) cv2.imshow("out",img) #show image, exit program if we press esc if cv2.waitKey(1) == 27: diff --git a/bin/parameters.py b/bin/parameters.py index 71511af..b052008 100644 --- a/bin/parameters.py +++ b/bin/parameters.py @@ -49,6 +49,8 @@ def __init__(self) -> None: self.img_rot_dict_rev = {None: 0, cv2.ROTATE_90_CLOCKWISE: 1, cv2.ROTATE_180: 2, cv2.ROTATE_90_COUNTERCLOCKWISE: 3} self.paused = False + + self.flip = False self.load_params() @@ -111,6 +113,12 @@ def save_params(self): param["rotx"] = self.global_rot_x.as_euler('zyx', degrees=True)[2] param["rotz"] = self.global_rot_z.as_euler('zyx', degrees=True)[0] param["scale"] = self.posescale + + param["calibrot"] = self.calib_rot + param["calibtilt"] = self.calib_tilt + param["calibscale"] = self.calib_scale + + param["flip"] = self.flip with open("saved_params.json", "w") as f: json.dump(param, f) @@ -130,6 +138,12 @@ def load_params(self): self.global_rot_x = R.from_euler('x',param["rotx"],degrees=True) self.global_rot_z = R.from_euler('z',param["rotz"],degrees=True) self.posescale = param["scale"] + + self.calib_rot = param["calibrot"] + self.calib_tilt = param["calibtilt"] + self.calib_scale = param["calibscale"] + + self.flip = param["flip"] except: print("Save file not found, will be created after you exit the program.") From fc99d2da2bacf253b037df498f84217aeefa2b59 Mon Sep 17 00:00:00 2001 From: ju1ce Date: Mon, 28 Feb 2022 17:37:21 +0100 Subject: [PATCH 09/20] Added two profiles for smoothing parameters --- bin/inference_gui.py | 41 +++++++++++++++++++++++++++++++++-------- bin/parameters.py | 40 +++++++++++++++++++++++++++++----------- 2 files changed, 62 insertions(+), 19 deletions(-) diff --git a/bin/inference_gui.py b/bin/inference_gui.py index 24bc6b6..7a3d81a 100644 --- a/bin/inference_gui.py +++ b/bin/inference_gui.py @@ -52,6 +52,17 @@ def __init__(self, root, params, *args, **kwargs): # pause tracking tk.Button(self.root, text='Pause/Unpause tracking', command=self.pause_tracking).pack() + + + + frame_profile = tk.Frame(self.root) + frame_profile.pack() + tk.Label(frame_profile, text=" ", width = 20).pack(side='left') + tk.Label(frame_profile, text="Profile 1", width = 10).pack(side='left') + tk.Label(frame_profile, text=" ", width = 5).pack(side='left') + tk.Label(frame_profile, text="Profile 2", width = 10).pack(side='left') + tk.Label(frame_profile, text=" ", width = 5).pack(side='left') + tk.Label(frame_profile, text=" ", width = 5).pack(side='left') # smoothing frame4 = tk.Frame(self.root) @@ -195,11 +206,18 @@ def calibrate_scale_frame(self, frame): def change_smooothing_frame(self, frame): tk.Label(frame, text="Smoothing window:", width = 20).pack(side='left') - smoothingtext = tk.Entry(frame, width = 10) - smoothingtext.pack(side='left') - smoothingtext.insert(0, self.params.smoothing) + smoothingtext1 = tk.Entry(frame, width = 10) + smoothingtext1.pack(side='left') + smoothingtext1.insert(0, self.params.smoothing_1) - tk.Button(frame, text='Update', command=lambda *args: self.params.change_smoothing(float(smoothingtext.get()))).pack(side='left') + tk.Button(frame, text='Update', command=lambda *args: self.params.change_smoothing(float(smoothingtext1.get()),1)).pack(side='left') + + smoothingtext2 = tk.Entry(frame, width = 10) + smoothingtext2.pack(side='left') + smoothingtext2.insert(0, self.params.smoothing_2) + + tk.Button(frame, text='Update', command=lambda *args: self.params.change_smoothing(float(smoothingtext2.get()),2)).pack(side='left') + tk.Button(frame, text='Disable', command=lambda *args: self.params.change_smoothing(0.0)).pack(side='left') @@ -215,11 +233,18 @@ def change_cam_lat_frame(self, frame): def change_add_smoothing_frame(self, frame): tk.Label(frame, text="Additional smoothing:", width = 20).pack(side='left') - lat = tk.Entry(frame, width = 10) - lat.pack(side='left') - lat.insert(0, self.params.additional_smoothing) + lat1 = tk.Entry(frame, width = 10) + lat1.pack(side='left') + lat1.insert(0, self.params.additional_smoothing_1) - tk.Button(frame, text='Update', command=lambda *args: self.params.change_additional_smoothing(float(lat.get()))).pack(side='left') + tk.Button(frame, text='Update', command=lambda *args: self.params.change_additional_smoothing(float(lat1.get()),1)).pack(side='left') + + lat2 = tk.Entry(frame, width = 10) + lat2.pack(side='left') + lat2.insert(0, self.params.additional_smoothing_2) + + tk.Button(frame, text='Update', command=lambda *args: self.params.change_additional_smoothing(float(lat2.get()),2)).pack(side='left') + tk.Button(frame, text='Disable', command=lambda *args: self.params.change_additional_smoothing(0.0)).pack(side='left') diff --git a/bin/parameters.py b/bin/parameters.py index b052008..c1ab41b 100644 --- a/bin/parameters.py +++ b/bin/parameters.py @@ -19,8 +19,10 @@ def __init__(self) -> None: #self.camera_latency = param["camlatency"] #self.smoothing = param["smooth"] self.camera_latency = 0.1 - self.smoothing = 0.5 - self.additional_smoothing = 0 + self.smoothing_1 = 0.5 + self.additional_smoothing_1 = 0 + self.smoothing_2 = 0.5 + self.additional_smoothing_2 = 0 self.feet_rotation = param["feetrot"] self.use_hands = param["use_hands"] self.ignore_hip = param["ignore_hip"] @@ -33,8 +35,6 @@ def __init__(self) -> None: self.calib_tilt = True self.calib_scale = True - self.prev_smoothing = self.smoothing - self.recalibrate = False self.global_rot_y = R.from_euler('y',0,degrees=True) #default rotations, for 0 degrees around y and x @@ -53,7 +53,11 @@ def __init__(self) -> None: self.flip = False self.load_params() - + + self.smoothing = self.smoothing_1 + self.additional_smoothing = self.additional_smoothing_1 + + #self.prev_smoothing = self.smoothing def change_recalibrate(self): self.recalibrate = True @@ -84,14 +88,24 @@ def change_img_rot(self, val): self.rotate_image = self.img_rot_dict[val] - def change_smoothing(self, val): + def change_smoothing(self, val, paramid = 0): print(f"Changed smoothing value to {val}") self.smoothing = val - def change_additional_smoothing(self, val): + if paramid == 1: + self.smoothing_1 = val + if paramid == 2: + self.smoothing_2 = val + + def change_additional_smoothing(self, val, paramid = 0): print(f"Changed additional smoothing value to {val}") self.additional_smoothing = val + if paramid == 1: + self.additional_smoothing_1 = val + if paramid == 2: + self.additional_smoothing_2 = val + def change_camera_latency(self, val): print(f"Changed camera latency to {val}") self.camera_latency = val @@ -104,10 +118,12 @@ def ready2exit(self): def save_params(self): param = {} param["rotate"] = self.img_rot_dict_rev[self.rotate_image] - param["smooth"] = self.smoothing + param["smooth1"] = self.smoothing_1 + param["smooth2"] = self.smoothing_2 param["camlatency"] = self.camera_latency - param["addsmooth"] = self.additional_smoothing + param["addsmooth1"] = self.additional_smoothing_1 + param["addsmooth2"] = self.additional_smoothing_2 param["roty"] = self.global_rot_y.as_euler('zyx', degrees=True)[1] param["rotx"] = self.global_rot_x.as_euler('zyx', degrees=True)[2] @@ -130,9 +146,11 @@ def load_params(self): param = json.load(f) self.rotate_image = self.img_rot_dict[param["rotate"]] - self.smoothing = param["smooth"] + self.smoothing_1 = param["smooth1"] + self.smoothing_2 = param["smooth2"] self.camera_latency = param["camlatency"] - self.additional_smoothing = param["addsmooth"] + self.additional_smoothing_1 = param["addsmooth1"] + self.additional_smoothing_2 = param["addsmooth2"] self.global_rot_y = R.from_euler('y',param["roty"],degrees=True) self.global_rot_x = R.from_euler('x',param["rotx"],degrees=True) From 30306b005f630772ccd6e163239047817a78b0fc Mon Sep 17 00:00:00 2001 From: ju1ce Date: Mon, 28 Feb 2022 18:02:48 +0100 Subject: [PATCH 10/20] added mediapipe parameters to gui --- bin/init_gui.py | 33 +++++++++++++++++++++++++++++++++ bin/mediapipepose.py | 5 +++-- bin/parameters.py | 4 ++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/bin/init_gui.py b/bin/init_gui.py index 3880266..c7047ab 100644 --- a/bin/init_gui.py +++ b/bin/init_gui.py @@ -47,6 +47,13 @@ def getparams(): param["camera_width"] = 640 if "camera_height" not in param: param["camera_height"] = 480 + if "model_complexity" not in param: + param["model_complexity"] = 1 + if "smooth_landmarks" not in param: + param["smooth_landmarks"] = True + if "min_tracking_confidence" not in param: + param["min_tracking_confidence"] = 0.5 + window = tk.Tk() @@ -138,6 +145,24 @@ def getparams(): skeleton_check = tk.Checkbutton(text = "DEV: preview skeleton in VR", variable = varskel) skeleton_check.pack() + tk.Label(text="-"*50, width = 50).pack() + + tk.Label(text="[ADVANCED] MediaPipe estimator parameters:", width = 50).pack() + + tk.Label(text="Model complexity:", width = 50).pack() + modelc = tk.Entry(width = 20) + modelc.pack() + modelc.insert(0,param["model_complexity"]) + + varmsmooth = tk.IntVar(value = param["smooth_landmarks"]) + msmooth_check = tk.Checkbutton(text = "Smooth landmarks", variable = varmsmooth) + msmooth_check.pack() + + tk.Label(text="Min tracking confidence:", width = 50).pack() + trackc = tk.Entry(width = 20) + trackc.pack() + trackc.insert(0,param["min_tracking_confidence"]) + tk.Button(text='Save and continue', command=window.quit).pack() window.mainloop() @@ -156,6 +181,10 @@ def getparams(): camera_settings = bool(varcamsettings.get()) camera_height = camheight.get() camera_width = camwidth.get() + + mp_smoothing = bool(varmsmooth.get()) + model_complexity = int(modelc.get()) + min_tracking_confidence = float(trackc.get()) param = {} param["camid"] = cameraid @@ -174,6 +203,10 @@ def getparams(): param["camera_height"] = camera_height param["camera_width"] = camera_width + param["model_complexity"] = model_complexity + param["smooth_landmarks"] = mp_smoothing + param["min_tracking_confidence"] = min_tracking_confidence + pickle.dump(param,open("params.p","wb")) window.destroy() diff --git a/bin/mediapipepose.py b/bin/mediapipepose.py index cda569b..1e78a90 100644 --- a/bin/mediapipepose.py +++ b/bin/mediapipepose.py @@ -132,9 +132,10 @@ def camera_thread_fun(): print("Starting pose detector...") pose = mp_pose.Pose( #create our detector. These are default parameters as used in the tutorial. + model_complexity=params.model, min_detection_confidence=0.5, - min_tracking_confidence=0.5, - model_complexity=1) + min_tracking_confidence=params.min_tracking_confidence, + smooth_landmarks=params.smooth_landmarks) def shutdown(): diff --git a/bin/parameters.py b/bin/parameters.py index c1ab41b..4d5dec2 100644 --- a/bin/parameters.py +++ b/bin/parameters.py @@ -6,6 +6,10 @@ class Parameters(): def __init__(self) -> None: param = getparams() + + self.model = param["model_complexity"] + self.smooth_landmarks = param["smooth_landmarks"] + self.min_tracking_confidence = param["min_tracking_confidence"] #PARAMETERS: #model = 1 #TODO: add parameter for which model size to use From 422fd6e728e0ac3fcbecdab95241c3c03b9f2f01 Mon Sep 17 00:00:00 2001 From: ju1ce Date: Mon, 28 Feb 2022 18:33:07 +0100 Subject: [PATCH 11/20] added option to print inference times to console --- bin/inference_gui.py | 12 +++++++++++- bin/mediapipepose.py | 6 +++++- bin/parameters.py | 11 +++++++++-- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/bin/inference_gui.py b/bin/inference_gui.py index 7a3d81a..63dedf5 100644 --- a/bin/inference_gui.py +++ b/bin/inference_gui.py @@ -16,7 +16,7 @@ def __init__(self, root, params, *args, **kwargs): # calibrate rotation self.calib_rot_var = tk.BooleanVar(value=self.params.calib_rot) self.calib_flip_var = tk.BooleanVar(value=self.params.flip) - self.rot_y_var = tk.DoubleVar(value=self.params.global_rot_y.as_euler('zyx', degrees=True)[1]) + self.rot_y_var = tk.DoubleVar(value=180-self.params.global_rot_y.as_euler('zyx', degrees=True)[1]) frame1 = tk.Frame(self.root) frame1.pack() @@ -83,6 +83,10 @@ def __init__(self, root, params, *args, **kwargs): frame5 = tk.Frame(self.root) frame5.pack() self.change_image_rotation_frame(frame5) + + self.log_frametime_var = tk.BooleanVar(value=self.params.log_frametime) + log_frametime_check = tk.Checkbutton(self.root, text="Log frametimes to console", variable=self.log_frametime_var, command=self.change_log_frametime) + log_frametime_check.pack() # exit tk.Button(self.root, text='Press to exit', command=self.params.ready2exit).pack() @@ -91,6 +95,12 @@ def __init__(self, root, params, *args, **kwargs): #self.root.after(0, self.set_rot_x_var) + def change_log_frametime(self): + self.params.log_frametime = self.log_frametime_var.get() + if self.params.log_frametime: + print("Enabled frametime logging") + else: + print("Disabled frametime logging") def set_rot_y_var(self): angle = -(180+self.params.global_rot_y.as_euler('zyx', degrees=True)[1]) diff --git a/bin/mediapipepose.py b/bin/mediapipepose.py index 1e78a90..1a46bda 100644 --- a/bin/mediapipepose.py +++ b/bin/mediapipepose.py @@ -273,10 +273,14 @@ def shutdown(): joint = pose3d[i] - offset #if previewing skeleton, send the position of each keypoint to steamvr without rotation if use_steamvr: sendToSteamVR(f"updatepose {i} {joint[0]} {joint[1]} {joint[2] - 2} 1 0 0 0 {params.camera_latency} 0.8") - + + #print(f"Inference time: {time.time()-t0}\nSmoothing value: {smoothing}\n") #print how long it took to detect and calculate everything inference_time = time.time() - t0 + if params.log_frametime: + print(f"Inference time: {inference_time}") + img = cv2.cvtColor(img,cv2.COLOR_RGB2BGR) #convert back to bgr and draw the pose mp_drawing.draw_landmarks( img, results.pose_landmarks, mp_pose.POSE_CONNECTIONS) diff --git a/bin/parameters.py b/bin/parameters.py index 4d5dec2..26516d6 100644 --- a/bin/parameters.py +++ b/bin/parameters.py @@ -55,6 +55,8 @@ def __init__(self) -> None: self.paused = False self.flip = False + + self.log_frametime = False self.load_params() @@ -129,7 +131,8 @@ def save_params(self): param["addsmooth1"] = self.additional_smoothing_1 param["addsmooth2"] = self.additional_smoothing_2 - param["roty"] = self.global_rot_y.as_euler('zyx', degrees=True)[1] + #if self.flip: + param["roty"] = 180-self.global_rot_y.as_euler('zyx', degrees=True)[1] param["rotx"] = self.global_rot_x.as_euler('zyx', degrees=True)[2] param["rotz"] = self.global_rot_z.as_euler('zyx', degrees=True)[0] param["scale"] = self.posescale @@ -139,7 +142,9 @@ def save_params(self): param["calibscale"] = self.calib_scale param["flip"] = self.flip - + + print(param["roty"]) + with open("saved_params.json", "w") as f: json.dump(param, f) @@ -149,6 +154,8 @@ def load_params(self): with open("saved_params.json", "r") as f: param = json.load(f) + print(param["roty"]) + self.rotate_image = self.img_rot_dict[param["rotate"]] self.smoothing_1 = param["smooth1"] self.smoothing_2 = param["smooth2"] From af09c0a9d35a990bc3f6ff33db1ea166d96abd12 Mon Sep 17 00:00:00 2001 From: ju1ce Date: Mon, 28 Feb 2022 18:57:27 +0100 Subject: [PATCH 12/20] added neck offset to inference gui --- bin/inference_gui.py | 23 +++++++++++++++++++++++ bin/init_gui.py | 14 ++++++++------ bin/mediapipepose.py | 3 ++- bin/parameters.py | 9 ++++++++- 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/bin/inference_gui.py b/bin/inference_gui.py index 63dedf5..8d2e46e 100644 --- a/bin/inference_gui.py +++ b/bin/inference_gui.py @@ -84,6 +84,13 @@ def __init__(self, root, params, *args, **kwargs): frame5.pack() self.change_image_rotation_frame(frame5) + # neck offset + + frame6 = tk.Frame(self.root) + frame6.pack() + self.change_neck_offset_frame(frame6) + + #frametime log self.log_frametime_var = tk.BooleanVar(value=self.params.log_frametime) log_frametime_check = tk.Checkbutton(self.root, text="Log frametimes to console", variable=self.log_frametime_var, command=self.change_log_frametime) log_frametime_check.pack() @@ -93,7 +100,23 @@ def __init__(self, root, params, *args, **kwargs): #self.root.after(0, self.set_rot_y_var) #self.root.after(0, self.set_rot_x_var) + + def change_neck_offset_frame(self,frame): + tk.Label(frame, text="HMD to neck offset:", width = 20).pack(side='left') + + text1 = tk.Entry(frame, width = 5) + text1.pack(side='left') + text1.insert(0, self.params.hmd_to_neck_offset[0]) + + text2 = tk.Entry(frame, width = 5) + text2.pack(side='left') + text2.insert(0, self.params.hmd_to_neck_offset[1]) + + text3 = tk.Entry(frame, width = 5) + text3.pack(side='left') + text3.insert(0, self.params.hmd_to_neck_offset[2]) + tk.Button(frame, text='Update', command=lambda *args: self.params.change_neck_offset(float(text1.get()),float(text2.get()),float(text3.get()))).pack(side='left') def change_log_frametime(self): self.params.log_frametime = self.log_frametime_var.get() diff --git a/bin/init_gui.py b/bin/init_gui.py index c7047ab..c4bc2ee 100644 --- a/bin/init_gui.py +++ b/bin/init_gui.py @@ -83,10 +83,12 @@ def getparams(): maximgsize.pack() maximgsize.insert(0,param["imgsize"]) - tk.Label(text="Offset of HMD to neck:", width = 50).pack() - hmdoffsettext = tk.Entry(width = 50) - hmdoffsettext.pack() - hmdoffsettext.insert(0," ".join(map(str,param["neckoffset"]))) + if False: + + tk.Label(text="Offset of HMD to neck:", width = 50).pack() + hmdoffsettext = tk.Entry(width = 50) + hmdoffsettext.pack() + hmdoffsettext.insert(0," ".join(map(str,param["neckoffset"]))) if False: tk.Label(text="Smoothing:", width = 50).pack() @@ -169,7 +171,7 @@ def getparams(): cameraid = camid.get() maximgsize = int(maximgsize.get()) - hmd_to_neck_offset = [float(val) for val in hmdoffsettext.get().split(" ")] + #hmd_to_neck_offset = [float(val) for val in hmdoffsettext.get().split(" ")] preview_skeleton = bool(varskel.get()) dont_wait_hmd = False #bool(varhmdwait.get()) @@ -189,7 +191,7 @@ def getparams(): param = {} param["camid"] = cameraid param["imgsize"] = maximgsize - param["neckoffset"] = hmd_to_neck_offset + #param["neckoffset"] = hmd_to_neck_offset param["prevskel"] = preview_skeleton param["waithmd"] = dont_wait_hmd diff --git a/bin/mediapipepose.py b/bin/mediapipepose.py index 1a46bda..92de04f 100644 --- a/bin/mediapipepose.py +++ b/bin/mediapipepose.py @@ -135,7 +135,8 @@ def camera_thread_fun(): model_complexity=params.model, min_detection_confidence=0.5, min_tracking_confidence=params.min_tracking_confidence, - smooth_landmarks=params.smooth_landmarks) + smooth_landmarks=params.smooth_landmarks, + static_image_mode=False) def shutdown(): diff --git a/bin/parameters.py b/bin/parameters.py index 26516d6..2ebb4da 100644 --- a/bin/parameters.py +++ b/bin/parameters.py @@ -16,7 +16,7 @@ def __init__(self) -> None: self.maximgsize = param["imgsize"] #to prevent working with huge images, images that have one axis larger than this value will be downscaled. self.cameraid = param["camid"] #to use with an usb or virtual webcam. If 0 doesnt work/opens wrong camera, try numbers 1-5 or so #cameraid = "http://192.168.1.102:8080/video" #to use ip webcam, uncomment this line and change to your ip - self.hmd_to_neck_offset = param["neckoffset"] #offset of your hmd to the base of your neck, to ensure the tracking is stable even if you look around. Default is 20cm down, 10cm back. + self.hmd_to_neck_offset = [0,-0.2,0.1] #offset of your hmd to the base of your neck, to ensure the tracking is stable even if you look around. Default is 20cm down, 10cm back. self.preview_skeleton = param["prevskel"] #if True, whole skeleton will appear in vr 2 meters in front of you. Good to visualize if everything is working self.dont_wait_hmd = param["waithmd"] #dont wait for movement from hmd, start inference immediately. self.rotate_image = 0 # cv2.ROTATE_90_CLOCKWISE # cv2.ROTATE_90_COUTERCLOCKWISE # cv2.ROTATE_180 # None # if you want, rotate the camera @@ -116,6 +116,9 @@ def change_camera_latency(self, val): print(f"Changed camera latency to {val}") self.camera_latency = val + def change_neck_offset(self,x,y,z): + print(f"Hmd to neck offset changed to: [{x},{y},{z}]") + self.hmd_to_neck_offset = [x,y,z] def ready2exit(self): self.exit_ready = True @@ -143,6 +146,8 @@ def save_params(self): param["flip"] = self.flip + param["hmd_to_neck_offset"] = self.hmd_to_neck_offset + print(param["roty"]) with open("saved_params.json", "w") as f: @@ -172,6 +177,8 @@ def load_params(self): self.calib_tilt = param["calibtilt"] self.calib_scale = param["calibscale"] + self.hmd_to_neck_offset = param["hmd_to_neck_offset"] + self.flip = param["flip"] except: print("Save file not found, will be created after you exit the program.") From d997b3f9152ce87111fc512f6b2384e418e5b743 Mon Sep 17 00:00:00 2001 From: ju1ce Date: Mon, 28 Feb 2022 19:52:16 +0100 Subject: [PATCH 13/20] added static image mode mediapipe option --- bin/init_gui.py | 9 ++++++++- bin/mediapipepose.py | 2 +- bin/parameters.py | 1 + 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/bin/init_gui.py b/bin/init_gui.py index c4bc2ee..9f61531 100644 --- a/bin/init_gui.py +++ b/bin/init_gui.py @@ -53,7 +53,8 @@ def getparams(): param["smooth_landmarks"] = True if "min_tracking_confidence" not in param: param["min_tracking_confidence"] = 0.5 - + if "static_image" not in param: + param["static_image"] = False window = tk.Tk() @@ -164,6 +165,10 @@ def getparams(): trackc = tk.Entry(width = 20) trackc.pack() trackc.insert(0,param["min_tracking_confidence"]) + + varstatic = tk.IntVar(value = param["static_image"]) + static_check = tk.Checkbutton(text = "Static image mode", variable = varstatic) + static_check.pack() tk.Button(text='Save and continue', command=window.quit).pack() @@ -187,6 +192,7 @@ def getparams(): mp_smoothing = bool(varmsmooth.get()) model_complexity = int(modelc.get()) min_tracking_confidence = float(trackc.get()) + static_image = bool(varstatic.get()) param = {} param["camid"] = cameraid @@ -207,6 +213,7 @@ def getparams(): param["model_complexity"] = model_complexity param["smooth_landmarks"] = mp_smoothing + param["static_image"] = static_image param["min_tracking_confidence"] = min_tracking_confidence pickle.dump(param,open("params.p","wb")) diff --git a/bin/mediapipepose.py b/bin/mediapipepose.py index 92de04f..d1f391c 100644 --- a/bin/mediapipepose.py +++ b/bin/mediapipepose.py @@ -136,7 +136,7 @@ def camera_thread_fun(): min_detection_confidence=0.5, min_tracking_confidence=params.min_tracking_confidence, smooth_landmarks=params.smooth_landmarks, - static_image_mode=False) + static_image_mode=params.static_image) def shutdown(): diff --git a/bin/parameters.py b/bin/parameters.py index 2ebb4da..e2dd45c 100644 --- a/bin/parameters.py +++ b/bin/parameters.py @@ -10,6 +10,7 @@ def __init__(self) -> None: self.model = param["model_complexity"] self.smooth_landmarks = param["smooth_landmarks"] self.min_tracking_confidence = param["min_tracking_confidence"] + self.static_image = param["static_image"] #PARAMETERS: #model = 1 #TODO: add parameter for which model size to use From c8b97a2fc57d01f749e3ad79c280e0b0bcc77fd4 Mon Sep 17 00:00:00 2001 From: ju1ce Date: Tue, 1 Mar 2022 14:04:43 +0100 Subject: [PATCH 14/20] hidden some parameters behind an advanced mode --- bin/inference_gui.py | 46 ++++++++++--------- bin/init_gui.py | 106 +++++++++++++++++++++++++++++-------------- bin/parameters.py | 13 ++++-- 3 files changed, 105 insertions(+), 60 deletions(-) diff --git a/bin/inference_gui.py b/bin/inference_gui.py index 8d2e46e..696755a 100644 --- a/bin/inference_gui.py +++ b/bin/inference_gui.py @@ -54,15 +54,16 @@ def __init__(self, root, params, *args, **kwargs): command=self.pause_tracking).pack() - - frame_profile = tk.Frame(self.root) - frame_profile.pack() - tk.Label(frame_profile, text=" ", width = 20).pack(side='left') - tk.Label(frame_profile, text="Profile 1", width = 10).pack(side='left') - tk.Label(frame_profile, text=" ", width = 5).pack(side='left') - tk.Label(frame_profile, text="Profile 2", width = 10).pack(side='left') - tk.Label(frame_profile, text=" ", width = 5).pack(side='left') - tk.Label(frame_profile, text=" ", width = 5).pack(side='left') + # show the Profile 1 profile 2 text: + if params.advanced: + frame_profile = tk.Frame(self.root) + frame_profile.pack() + tk.Label(frame_profile, text=" ", width = 20).pack(side='left') + tk.Label(frame_profile, text="Profile 1", width = 10).pack(side='left') + tk.Label(frame_profile, text=" ", width = 5).pack(side='left') + tk.Label(frame_profile, text="Profile 2", width = 10).pack(side='left') + tk.Label(frame_profile, text=" ", width = 5).pack(side='left') + tk.Label(frame_profile, text=" ", width = 5).pack(side='left') # smoothing frame4 = tk.Frame(self.root) @@ -85,10 +86,10 @@ def __init__(self, root, params, *args, **kwargs): self.change_image_rotation_frame(frame5) # neck offset - - frame6 = tk.Frame(self.root) - frame6.pack() - self.change_neck_offset_frame(frame6) + if params.advanced: + frame6 = tk.Frame(self.root) + frame6.pack() + self.change_neck_offset_frame(frame6) #frametime log self.log_frametime_var = tk.BooleanVar(value=self.params.log_frametime) @@ -245,11 +246,12 @@ def change_smooothing_frame(self, frame): tk.Button(frame, text='Update', command=lambda *args: self.params.change_smoothing(float(smoothingtext1.get()),1)).pack(side='left') - smoothingtext2 = tk.Entry(frame, width = 10) - smoothingtext2.pack(side='left') - smoothingtext2.insert(0, self.params.smoothing_2) + if self.params.advanced: + smoothingtext2 = tk.Entry(frame, width = 10) + smoothingtext2.pack(side='left') + smoothingtext2.insert(0, self.params.smoothing_2) - tk.Button(frame, text='Update', command=lambda *args: self.params.change_smoothing(float(smoothingtext2.get()),2)).pack(side='left') + tk.Button(frame, text='Update', command=lambda *args: self.params.change_smoothing(float(smoothingtext2.get()),2)).pack(side='left') tk.Button(frame, text='Disable', command=lambda *args: self.params.change_smoothing(0.0)).pack(side='left') @@ -271,12 +273,12 @@ def change_add_smoothing_frame(self, frame): lat1.insert(0, self.params.additional_smoothing_1) tk.Button(frame, text='Update', command=lambda *args: self.params.change_additional_smoothing(float(lat1.get()),1)).pack(side='left') - - lat2 = tk.Entry(frame, width = 10) - lat2.pack(side='left') - lat2.insert(0, self.params.additional_smoothing_2) + if self.params.advanced: + lat2 = tk.Entry(frame, width = 10) + lat2.pack(side='left') + lat2.insert(0, self.params.additional_smoothing_2) - tk.Button(frame, text='Update', command=lambda *args: self.params.change_additional_smoothing(float(lat2.get()),2)).pack(side='left') + tk.Button(frame, text='Update', command=lambda *args: self.params.change_additional_smoothing(float(lat2.get()),2)).pack(side='left') tk.Button(frame, text='Disable', command=lambda *args: self.params.change_additional_smoothing(0.0)).pack(side='left') diff --git a/bin/init_gui.py b/bin/init_gui.py index 9f61531..1ee3b1f 100644 --- a/bin/init_gui.py +++ b/bin/init_gui.py @@ -2,6 +2,10 @@ import pickle import cv2 +def set_advanced(window,param): + param["switch_advanced"] = True + window.quit() + def getparams(): try: @@ -55,7 +59,9 @@ def getparams(): param["min_tracking_confidence"] = 0.5 if "static_image" not in param: param["static_image"] = False - + if "advanced" not in param: + param["advanced"] = False + window = tk.Tk() tk.Label(text="Camera IP or ID:", width = 50).pack() @@ -138,37 +144,45 @@ def getparams(): hip_check = tk.Checkbutton(text = "Don't use hip tracker", variable = varhip) hip_check.pack() - tk.Label(text="-"*50, width = 50).pack() + param["switch_advanced"] = False + if param["advanced"]: + tk.Button(text='Disable advanced mode', command=lambda *args: set_advanced(window,param)).pack() + else: + tk.Button(text='Enable advanced mode', command=lambda *args: set_advanced(window,param)).pack() - varhand = tk.IntVar(value = param["use_hands"]) - hand_check = tk.Checkbutton(text = "DEV: spawn trackers for hands", variable = varhand) - hand_check.pack() + tk.Label(text="-"*50, width = 50).pack() - varskel = tk.IntVar(value = param["prevskel"]) - skeleton_check = tk.Checkbutton(text = "DEV: preview skeleton in VR", variable = varskel) - skeleton_check.pack() + if param["advanced"]: + + varhand = tk.IntVar(value = param["use_hands"]) + hand_check = tk.Checkbutton(text = "DEV: spawn trackers for hands", variable = varhand) + hand_check.pack() + + varskel = tk.IntVar(value = param["prevskel"]) + skeleton_check = tk.Checkbutton(text = "DEV: preview skeleton in VR", variable = varskel) + skeleton_check.pack() - tk.Label(text="-"*50, width = 50).pack() + tk.Label(text="-"*50, width = 50).pack() - tk.Label(text="[ADVANCED] MediaPipe estimator parameters:", width = 50).pack() + tk.Label(text="[ADVANCED] MediaPipe estimator parameters:", width = 50).pack() - tk.Label(text="Model complexity:", width = 50).pack() - modelc = tk.Entry(width = 20) - modelc.pack() - modelc.insert(0,param["model_complexity"]) - - varmsmooth = tk.IntVar(value = param["smooth_landmarks"]) - msmooth_check = tk.Checkbutton(text = "Smooth landmarks", variable = varmsmooth) - msmooth_check.pack() - - tk.Label(text="Min tracking confidence:", width = 50).pack() - trackc = tk.Entry(width = 20) - trackc.pack() - trackc.insert(0,param["min_tracking_confidence"]) - - varstatic = tk.IntVar(value = param["static_image"]) - static_check = tk.Checkbutton(text = "Static image mode", variable = varstatic) - static_check.pack() + tk.Label(text="Model complexity:", width = 50).pack() + modelc = tk.Entry(width = 20) + modelc.pack() + modelc.insert(0,param["model_complexity"]) + + varmsmooth = tk.IntVar(value = param["smooth_landmarks"]) + msmooth_check = tk.Checkbutton(text = "Smooth landmarks", variable = varmsmooth) + msmooth_check.pack() + + tk.Label(text="Min tracking confidence:", width = 50).pack() + trackc = tk.Entry(width = 20) + trackc.pack() + trackc.insert(0,param["min_tracking_confidence"]) + + varstatic = tk.IntVar(value = param["static_image"]) + static_check = tk.Checkbutton(text = "Static image mode", variable = varstatic) + static_check.pack() tk.Button(text='Save and continue', command=window.quit).pack() @@ -177,22 +191,38 @@ def getparams(): cameraid = camid.get() maximgsize = int(maximgsize.get()) #hmd_to_neck_offset = [float(val) for val in hmdoffsettext.get().split(" ")] - preview_skeleton = bool(varskel.get()) + dont_wait_hmd = False #bool(varhmdwait.get()) #camera_latency = float(camlatencytext.get()) #smoothing = float(smoothingtext.get()) feet_rotation = bool(varfeet.get()) - use_hands = bool(varhand.get()) + ignore_hip = bool(varhip.get()) camera_settings = bool(varcamsettings.get()) camera_height = camheight.get() camera_width = camwidth.get() - mp_smoothing = bool(varmsmooth.get()) - model_complexity = int(modelc.get()) - min_tracking_confidence = float(trackc.get()) - static_image = bool(varstatic.get()) + if param["advanced"]: + preview_skeleton = bool(varskel.get()) + use_hands = bool(varhand.get()) + + mp_smoothing = bool(varmsmooth.get()) + model_complexity = int(modelc.get()) + min_tracking_confidence = float(trackc.get()) + static_image = bool(varstatic.get()) + else: + preview_skeleton = False + use_hands = False + + mp_smoothing = True + model_complexity = 1 + min_tracking_confidence = 0.5 + static_image = False + + switch_advanced = param["switch_advanced"] + + advanced = param["advanced"] param = {} param["camid"] = cameraid @@ -216,11 +246,19 @@ def getparams(): param["static_image"] = static_image param["min_tracking_confidence"] = min_tracking_confidence + if switch_advanced: + param["advanced"] = not advanced + else: + param["advanced"] = advanced + pickle.dump(param,open("params.p","wb")) window.destroy() - return param + if switch_advanced: + return None + else: + return param if __name__ == "__main__": print(getparams()) diff --git a/bin/parameters.py b/bin/parameters.py index e2dd45c..c6998e3 100644 --- a/bin/parameters.py +++ b/bin/parameters.py @@ -5,7 +5,11 @@ class Parameters(): def __init__(self) -> None: - param = getparams() + param = None + while param == None: + param = getparams() + + self.advanced = param["advanced"] self.model = param["model_complexity"] self.smooth_landmarks = param["smooth_landmarks"] @@ -149,7 +153,7 @@ def save_params(self): param["hmd_to_neck_offset"] = self.hmd_to_neck_offset - print(param["roty"]) + #print(param["roty"]) with open("saved_params.json", "w") as f: json.dump(param, f) @@ -160,7 +164,7 @@ def load_params(self): with open("saved_params.json", "r") as f: param = json.load(f) - print(param["roty"]) + #print(param["roty"]) self.rotate_image = self.img_rot_dict[param["rotate"]] self.smoothing_1 = param["smooth1"] @@ -178,7 +182,8 @@ def load_params(self): self.calib_tilt = param["calibtilt"] self.calib_scale = param["calibscale"] - self.hmd_to_neck_offset = param["hmd_to_neck_offset"] + if self.advanced: + self.hmd_to_neck_offset = param["hmd_to_neck_offset"] self.flip = param["flip"] except: From 942e77ea1b7dd82efb0c0a0acc852118ab760642 Mon Sep 17 00:00:00 2001 From: ju1ce Date: Wed, 2 Mar 2022 12:57:20 +0100 Subject: [PATCH 15/20] added option to mirror image --- bin/inference_gui.py | 4 ++++ bin/mediapipepose.py | 5 ++++- bin/parameters.py | 11 ++++++++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/bin/inference_gui.py b/bin/inference_gui.py index 696755a..91f7302 100644 --- a/bin/inference_gui.py +++ b/bin/inference_gui.py @@ -293,6 +293,10 @@ def change_image_rotation_frame(self, frame): rot_img_var.trace_add('write', callback=lambda var, index, mode: self.params.change_img_rot(rot_img_var.get())) + img_mirror_var = tk.BooleanVar(value=self.params.mirror) + img_mirror_check = tk.Checkbutton(frame, text="Mirror", variable=img_mirror_var, command=lambda *args: self.params.change_mirror(bool(img_mirror_var.get()))) + img_mirror_check.grid(row=0, column=5) + def autocalibrate(self): diff --git a/bin/mediapipepose.py b/bin/mediapipepose.py index d1f391c..2a8aa52 100644 --- a/bin/mediapipepose.py +++ b/bin/mediapipepose.py @@ -21,7 +21,7 @@ mp_drawing = mp.solutions.drawing_utils mp_pose = mp.solutions.pose -use_steamvr = True +use_steamvr = False print("Reading parameters...") @@ -186,6 +186,9 @@ def shutdown(): if params.rotate_image is not None: #if set, rotate the image img = cv2.rotate(img, params.rotate_image) + + if params.mirror: + img = cv2.flip(img,1) if max(img.shape) > params.maximgsize: #if set, ensure image does not exceed the given size. ratio = max(img.shape)/params.maximgsize diff --git a/bin/parameters.py b/bin/parameters.py index c6998e3..3b4bb44 100644 --- a/bin/parameters.py +++ b/bin/parameters.py @@ -62,6 +62,8 @@ def __init__(self) -> None: self.flip = False self.log_frametime = False + + self.mirror = False self.load_params() @@ -125,10 +127,13 @@ def change_neck_offset(self,x,y,z): print(f"Hmd to neck offset changed to: [{x},{y},{z}]") self.hmd_to_neck_offset = [x,y,z] + def change_mirror(self, mirror): + print(f"Image mirror set to {mirror}") + self.mirror = mirror + def ready2exit(self): self.exit_ready = True - def save_params(self): param = {} param["rotate"] = self.img_rot_dict_rev[self.rotate_image] @@ -153,6 +158,8 @@ def save_params(self): param["hmd_to_neck_offset"] = self.hmd_to_neck_offset + param["mirror"] = self.mirror + #print(param["roty"]) with open("saved_params.json", "w") as f: @@ -182,6 +189,8 @@ def load_params(self): self.calib_tilt = param["calibtilt"] self.calib_scale = param["calibscale"] + self.mirror = param["mirror"] + if self.advanced: self.hmd_to_neck_offset = param["hmd_to_neck_offset"] From 5b5c6764e64baaac08915aa9158141a81abe7759 Mon Sep 17 00:00:00 2001 From: ju1ce Date: Thu, 3 Mar 2022 15:39:45 +0100 Subject: [PATCH 16/20] Camera API no longer set to 700 if camera is IP camera --- bin/mediapipepose.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/bin/mediapipepose.py b/bin/mediapipepose.py index 2a8aa52..1424eae 100644 --- a/bin/mediapipepose.py +++ b/bin/mediapipepose.py @@ -21,7 +21,7 @@ mp_drawing = mp.solutions.drawing_utils mp_pose = mp.solutions.pose -use_steamvr = False +use_steamvr = True print("Reading parameters...") @@ -39,14 +39,15 @@ def camera_thread_fun(): if len(params.cameraid) <= 2: cameraid = int(params.cameraid) + if params.camera_settings: + cap = cv2.VideoCapture(cameraid, 700) + cap.set(cv2.CAP_PROP_SETTINGS,1) + else: + cap = cv2.VideoCapture(cameraid) else: cameraid = params.cameraid - - if params.camera_settings: - cap = cv2.VideoCapture(cameraid, 700) - cap.set(cv2.CAP_PROP_SETTINGS,1) - else: - cap = cv2.VideoCapture(cameraid) + cap = cv2.VideoCapture(cameraid) + #codec = 0x47504A4D #cap.set(cv2.CAP_PROP_FOURCC, codec) From 4c846c9a37c6e742ae0e5cf393091cadc1e1ca38 Mon Sep 17 00:00:00 2001 From: ju1ce Date: Thu, 3 Mar 2022 20:12:12 +0100 Subject: [PATCH 17/20] fixed calibration values not saving properly --- bin/inference_gui.py | 34 +++++++++++++++++----------------- bin/init_gui.py | 33 +++++++++++++++++++++------------ bin/parameters.py | 28 ++++++++++++++++++---------- 3 files changed, 56 insertions(+), 39 deletions(-) diff --git a/bin/inference_gui.py b/bin/inference_gui.py index 91f7302..e2db826 100644 --- a/bin/inference_gui.py +++ b/bin/inference_gui.py @@ -127,9 +127,9 @@ def change_log_frametime(self): print("Disabled frametime logging") def set_rot_y_var(self): - angle = -(180+self.params.global_rot_y.as_euler('zyx', degrees=True)[1]) + angle = self.params.euler_rot_y - if not self.params.flip: + if self.params.flip: angle += 180 #print("calculated angle from rot is:",angle) @@ -143,11 +143,11 @@ def set_rot_y_var(self): def set_rot_z_var(self): - self.rot_z_var.set(self.params.global_rot_z.as_euler('zyx', degrees=True)[0]+180) + self.rot_z_var.set(self.params.euler_rot_z) def set_rot_x_var(self): - self.rot_x_var.set(self.params.global_rot_x.as_euler('zyx', degrees=True)[2]+90) + self.rot_x_var.set(self.params.euler_rot_x) #self.root.after(0, self.set_rot_x_var) @@ -326,37 +326,37 @@ def autocalibrate(self): ## roll calibaration - value = np.arctan2(feet_middle[0],-feet_middle[1]) + value = np.arctan2(feet_middle[0],-feet_middle[1]) * 57.295779513 - print("Precalib z angle: ", value * 57.295779513) + print("Precalib z angle: ", value) - self.params.global_rot_z = R.from_euler('z',-value) + self.params.rot_change_z(-value+180) self.set_rot_z_var() for j in range(self.params.pose3d_og.shape[0]): self.params.pose3d_og[j] = self.params.global_rot_z.apply(self.params.pose3d_og[j]) feet_middle = (self.params.pose3d_og[0] + self.params.pose3d_og[5])/2 - value = np.arctan2(feet_middle[0],-feet_middle[1]) + value = np.arctan2(feet_middle[0],-feet_middle[1]) * 57.295779513 - print("Postcalib z angle: ", value * 57.295779513) + print("Postcalib z angle: ", value) ##tilt calibration - value = np.arctan2(feet_middle[2],-feet_middle[1]) + value = np.arctan2(feet_middle[2],-feet_middle[1]) * 57.295779513 - print("Precalib x angle: ", value * 57.295779513) + print("Precalib x angle: ", value) - self.params.global_rot_x = R.from_euler('x',value) + self.params.rot_change_x(value+90) self.set_rot_x_var() for j in range(self.params.pose3d_og.shape[0]): self.params.pose3d_og[j] = self.params.global_rot_x.apply(self.params.pose3d_og[j]) feet_middle = (self.params.pose3d_og[0] + self.params.pose3d_og[5])/2 - value = np.arctan2(feet_middle[2],-feet_middle[1]) + value = np.arctan2(feet_middle[2],-feet_middle[1]) * 57.295779513 - print("Postcalib x angle: ", value * 57.295779513) + print("Postcalib x angle: ", value) if use_steamvr and self.params.calib_rot: feet_rot = self.params.pose3d_og[0] - self.params.pose3d_og[5] @@ -371,11 +371,11 @@ def autocalibrate(self): print("Calibrate to value:", value * 57.295779513) - self.params.global_rot_y = R.from_euler('y',value) + self.params.rot_change_y(value) - angle = self.params.global_rot_y.as_euler('zyx', degrees=True) - print("angle from rot = ", -(180+angle[1]), "whole:",angle) + #angle = self.params.global_rot_y.as_euler('zyx', degrees=True) + #print("angle from rot = ", -(180+angle[1]), "whole:",angle) self.set_rot_y_var() diff --git a/bin/init_gui.py b/bin/init_gui.py index 1ee3b1f..41b18ed 100644 --- a/bin/init_gui.py +++ b/bin/init_gui.py @@ -16,7 +16,7 @@ def getparams(): if "camid" not in param: param["camid"] = 'http://192.168.1.103:8080/video' if "imgsize" not in param: - param["imgsize"] = 800 + param["imgsize"] = 640 if "neckoffset" not in param: param["neckoffset"] = [0.0, -0.2, 0.1] if "prevskel" not in param: @@ -81,14 +81,19 @@ def getparams(): camheight.pack() camheight.insert(0,param["camera_height"]) + tk.Label(text="NOTE: Opening camera settings may change camera behaviour. \nSome cameras may only work with this enabled, some only with this \ndisabled, and it may change which camera ID you have to use.", width = 55).pack() + varcamsettings = tk.IntVar(value = param["camera_settings"]) cam_settings_check = tk.Checkbutton(text = "Attempt to open camera settings", variable = varcamsettings) cam_settings_check.pack() - tk.Label(text="Maximum image size:", width = 50).pack() - maximgsize = tk.Entry(width = 20) - maximgsize.pack() - maximgsize.insert(0,param["imgsize"]) + if param["advanced"]: + tk.Label(text="Maximum image size:", width = 50).pack() + maximgsize = tk.Entry(width = 20) + maximgsize.pack() + maximgsize.insert(0,param["imgsize"]) + + tk.Label(text="-"*50, width = 50).pack() if False: @@ -144,12 +149,6 @@ def getparams(): hip_check = tk.Checkbutton(text = "Don't use hip tracker", variable = varhip) hip_check.pack() - param["switch_advanced"] = False - if param["advanced"]: - tk.Button(text='Disable advanced mode', command=lambda *args: set_advanced(window,param)).pack() - else: - tk.Button(text='Enable advanced mode', command=lambda *args: set_advanced(window,param)).pack() - tk.Label(text="-"*50, width = 50).pack() if param["advanced"]: @@ -183,13 +182,19 @@ def getparams(): varstatic = tk.IntVar(value = param["static_image"]) static_check = tk.Checkbutton(text = "Static image mode", variable = varstatic) static_check.pack() + + param["switch_advanced"] = False + if param["advanced"]: + tk.Label(text="-"*50, width = 50).pack() + tk.Button(text='Disable advanced mode', command=lambda *args: set_advanced(window,param)).pack() + else: + tk.Button(text='Enable advanced mode', command=lambda *args: set_advanced(window,param)).pack() tk.Button(text='Save and continue', command=window.quit).pack() window.mainloop() cameraid = camid.get() - maximgsize = int(maximgsize.get()) #hmd_to_neck_offset = [float(val) for val in hmdoffsettext.get().split(" ")] dont_wait_hmd = False #bool(varhmdwait.get()) @@ -204,6 +209,8 @@ def getparams(): camera_width = camwidth.get() if param["advanced"]: + maximgsize = int(maximgsize.get()) + preview_skeleton = bool(varskel.get()) use_hands = bool(varhand.get()) @@ -212,6 +219,8 @@ def getparams(): min_tracking_confidence = float(trackc.get()) static_image = bool(varstatic.get()) else: + maximgsize = 640 + preview_skeleton = False use_hands = False diff --git a/bin/parameters.py b/bin/parameters.py index 3b4bb44..06663db 100644 --- a/bin/parameters.py +++ b/bin/parameters.py @@ -45,10 +45,11 @@ def __init__(self) -> None: self.calib_scale = True self.recalibrate = False - - self.global_rot_y = R.from_euler('y',0,degrees=True) #default rotations, for 0 degrees around y and x - self.global_rot_x = R.from_euler('x',0,degrees=True) - self.global_rot_z = R.from_euler('z',0,degrees=True) + + #rotations in degrees! + self.euler_rot_y = 0 + self.euler_rot_x = 0 + self.euler_rot_z = 0 self.posescale = 1 @@ -67,6 +68,10 @@ def __init__(self) -> None: self.load_params() + self.global_rot_y = R.from_euler('y',self.euler_rot_y,degrees=True) #default rotations, for 0 degrees around y and x + self.global_rot_x = R.from_euler('x',self.euler_rot_x-90,degrees=True) + self.global_rot_z = R.from_euler('z',self.euler_rot_z-180,degrees=True) + self.smoothing = self.smoothing_1 self.additional_smoothing = self.additional_smoothing_1 @@ -78,15 +83,18 @@ def change_recalibrate(self): def rot_change_y(self, value): #callback functions. Whenever the value on sliders are changed, they are called print(f"Changed y rotation value to {value}") + self.euler_rot_y = value self.global_rot_y = R.from_euler('y',value,degrees=True) #and the rotation is updated with the new value. def rot_change_x(self, value): print(f"Changed x rotation value to {value}") + self.euler_rot_x = value self.global_rot_x = R.from_euler('x',value-90,degrees=True) def rot_change_z(self, value): print(f"Changed z rotation value to {value}") + self.euler_rot_z = value self.global_rot_z = R.from_euler('z',value-180,degrees=True) @@ -145,9 +153,9 @@ def save_params(self): param["addsmooth2"] = self.additional_smoothing_2 #if self.flip: - param["roty"] = 180-self.global_rot_y.as_euler('zyx', degrees=True)[1] - param["rotx"] = self.global_rot_x.as_euler('zyx', degrees=True)[2] - param["rotz"] = self.global_rot_z.as_euler('zyx', degrees=True)[0] + param["roty"] = self.euler_rot_y + param["rotx"] = self.euler_rot_x + param["rotz"] = self.euler_rot_z param["scale"] = self.posescale param["calibrot"] = self.calib_rot @@ -180,9 +188,9 @@ def load_params(self): self.additional_smoothing_1 = param["addsmooth1"] self.additional_smoothing_2 = param["addsmooth2"] - self.global_rot_y = R.from_euler('y',param["roty"],degrees=True) - self.global_rot_x = R.from_euler('x',param["rotx"],degrees=True) - self.global_rot_z = R.from_euler('z',param["rotz"],degrees=True) + self.euler_rot_y = param["roty"] + self.euler_rot_x = param["rotx"] + self.euler_rot_z = param["rotz"] self.posescale = param["scale"] self.calib_rot = param["calibrot"] From 9bcb50b7c72823bb7b511ea0b5ef875e168e57d2 Mon Sep 17 00:00:00 2001 From: ju1ce Date: Thu, 3 Mar 2022 20:48:30 +0100 Subject: [PATCH 18/20] renamed ignore hip parameter --- bin/init_gui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/init_gui.py b/bin/init_gui.py index 41b18ed..149bdcd 100644 --- a/bin/init_gui.py +++ b/bin/init_gui.py @@ -146,7 +146,7 @@ def getparams(): rot_check.pack() varhip = tk.IntVar(value = param["ignore_hip"]) - hip_check = tk.Checkbutton(text = "Don't use hip tracker", variable = varhip) + hip_check = tk.Checkbutton(text = "Only spawn feet trackers", variable = varhip) hip_check.pack() tk.Label(text="-"*50, width = 50).pack() From 87c69938671d4b55de3132a73312cec2a2202686 Mon Sep 17 00:00:00 2001 From: ju1ce Date: Mon, 14 Mar 2022 18:57:12 +0100 Subject: [PATCH 19/20] added notes to not show when in advanced mode --- bin/init_gui.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/bin/init_gui.py b/bin/init_gui.py index 149bdcd..207b038 100644 --- a/bin/init_gui.py +++ b/bin/init_gui.py @@ -69,7 +69,8 @@ def getparams(): camid.pack() camid.insert(0,param["camid"]) - tk.Label(text="NOTE: Increasing resolution may decrease performance.\n Unless you have problems with opening camera, leave it at default.", width = 50).pack() + if not param["advanced"]: + tk.Label(text="NOTE: Increasing resolution may decrease performance.\n Unless you have problems with opening camera, leave it at default.", width = 50).pack() tk.Label(text="Camera width:", width = 50).pack() camwidth = tk.Entry(width = 20) @@ -81,7 +82,8 @@ def getparams(): camheight.pack() camheight.insert(0,param["camera_height"]) - tk.Label(text="NOTE: Opening camera settings may change camera behaviour. \nSome cameras may only work with this enabled, some only with this \ndisabled, and it may change which camera ID you have to use.", width = 55).pack() + if not param["advanced"]: + tk.Label(text="NOTE: Opening camera settings may change camera behaviour. \nSome cameras may only work with this enabled, some only with this \ndisabled, and it may change which camera ID you have to use.", width = 55).pack() varcamsettings = tk.IntVar(value = param["camera_settings"]) cam_settings_check = tk.Checkbutton(text = "Attempt to open camera settings", variable = varcamsettings) @@ -145,8 +147,11 @@ def getparams(): rot_check = tk.Checkbutton(text = "Enable automatic rotation calibration", variable = varrot) rot_check.pack() + if not param["advanced"]: + tk.Label(text="NOTE: VRChat requires a hip tracker. Only disable it if you \nuse another software for hip tracking, such as owoTrack.", width = 55).pack() + varhip = tk.IntVar(value = param["ignore_hip"]) - hip_check = tk.Checkbutton(text = "Only spawn feet trackers", variable = varhip) + hip_check = tk.Checkbutton(text = "Disable hip tracker", variable = varhip) hip_check.pack() tk.Label(text="-"*50, width = 50).pack() From 7674107775fa90b958faa9f4552bb1940da2ecf8 Mon Sep 17 00:00:00 2001 From: ju1ce Date: Mon, 14 Mar 2022 19:55:36 +0100 Subject: [PATCH 20/20] fixed calibration parameters not being read correctly --- bin/inference_gui.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/inference_gui.py b/bin/inference_gui.py index e2db826..9a2f390 100644 --- a/bin/inference_gui.py +++ b/bin/inference_gui.py @@ -16,7 +16,7 @@ def __init__(self, root, params, *args, **kwargs): # calibrate rotation self.calib_rot_var = tk.BooleanVar(value=self.params.calib_rot) self.calib_flip_var = tk.BooleanVar(value=self.params.flip) - self.rot_y_var = tk.DoubleVar(value=180-self.params.global_rot_y.as_euler('zyx', degrees=True)[1]) + self.rot_y_var = tk.DoubleVar(value=self.params.euler_rot_y) frame1 = tk.Frame(self.root) frame1.pack() @@ -26,8 +26,8 @@ def __init__(self, root, params, *args, **kwargs): # calibrate tilt self.calib_tilt_var = tk.BooleanVar(value=self.params.calib_tilt) - self.rot_x_var = tk.DoubleVar(value=self.params.global_rot_x.as_euler('zyx', degrees=True)[2]+90) - self.rot_z_var = tk.DoubleVar(value=self.params.global_rot_z.as_euler('zyx', degrees=True)[0]+180) + self.rot_x_var = tk.DoubleVar(value=self.params.euler_rot_x) + self.rot_z_var = tk.DoubleVar(value=self.params.euler_rot_z) frame2 = tk.Frame(self.root) frame2.pack() @@ -371,7 +371,7 @@ def autocalibrate(self): print("Calibrate to value:", value * 57.295779513) - self.params.rot_change_y(value) + self.params.rot_change_y(value * 57.295779513) #angle = self.params.global_rot_y.as_euler('zyx', degrees=True)