From 9515773ffe1ca26f2d4effd4b7fe2796c8f244e7 Mon Sep 17 00:00:00 2001
From: "lixin@20200"
+ +
+ ### Composing the Hand Based on the Anatomical Consistent Basis, we can also compose the hand from a given euler angles. :eyes: See: [manotorch/axislayer.py](manotorch/axislayer.py): `AxisLayerFK.compose` for details (FK: forward kinematics). -:runner: Run: [scripts/simple_compose.py](scripts/simple_compose.py), It demonstrates how we specify the euler angles of joint on the index finger and compose the hand in a deterministic way. +:runner: Run: [scripts/simple_compose.py](scripts/simple_compose.py), It shows how we specify the euler angles of joint on the index finger and compose the hand in a deterministic way. ```shell # transform order of right hand @@ -212,7 +215,7 @@ random_shape = torch.rand(batch_size, 10) # Generate random pose parameters, including 3 values for global axis-angle rotation random_pose = torch.rand(batch_size, 3 + ncomps) -# The mano_layer's output contains: +# The mano_layer's output contains: """ MANOOutput = namedtuple( "MANOOutput", @@ -234,5 +237,10 @@ mano_output: MANOOutput = mano_layer(random_pose, random_shape) verts = mano_output.verts # (B, 778, 3), root(center_joint) relative joints = mano_output.joints # (B, 21, 3), root relative transforms_abs = mano_output.transforms_abs # (B, 16, 4, 4), root relative -``` -For advance usages, please visit [scripts](./scripts/) for demonstrations. +``` + +### Advanced Usage + +| [Visualize](scripts/simple_app.py) | [Simple Compose](scripts/simple_compose.py) | [Error Correction](scripts/simple_anatomy_loss.py) | +| :--------------------------------: | :-----------------------------------------: | :------------------------------------------------: | +| ![](doc/axis.gif) | ![](doc/simple_compose.gif) | ![](doc/pose_correction.gif) | diff --git a/demo_axis_layer.py b/demo_axis_layer.py deleted file mode 100644 index 6c4a673..0000000 --- a/demo_axis_layer.py +++ /dev/null @@ -1,102 +0,0 @@ -from math import pi - -import numpy as np -import open3d as o3d -import torch -import tqdm - -from manotorch.axislayer import AxisAdaptiveLayer, AxisLayerFK -from manotorch.manolayer import ManoLayer, MANOOutput -from manotorch.utils.visutils import VizContext, create_coord_system_can - - -def main(): - viz_ctx = VizContext(non_block=True) - viz_ctx.init() - geometry_to_viz = dict( - hand_mesh_init=None, - hand_mesh_curr=None, - axis=None, - coord_system_list=None, - ) - - mano_layer = ManoLayer(rot_mode="axisang", - center_idx=9, - mano_assets_root="assets/mano", - use_pca=False, - flat_hand_mean=True) - hand_faces = mano_layer.th_faces # (NF, 3) - - axisIK = AxisLayerFK(mano_assets_root="assets/mano") - - # constructing the initial bending index fingers - global_aa = torch.zeros((1, 1, 3)) - composed_ee = torch.zeros((1, 16, 3)) - composed_ee[:, 1] = torch.tensor([0, 0, pi / 2]).unsqueeze(0) - composed_ee[:, 2] = torch.tensor([0, 0, pi / 2]).unsqueeze(0) - composed_ee[:, 3] = torch.tensor([0, 0, pi / 2]).unsqueeze(0) - composed_aa = axisIK.compose(composed_ee)[:, 1:, :].clone() # (B, 15, 3) - composed_aa.requires_grad_(True) - zero_shape = torch.zeros((1, 10)) - - param = [] - param.append({"params": [composed_aa]}) - optimizer = torch.optim.Adam(param, lr=1e-3) - scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=500, gamma=0.5) - proc_bar = tqdm.tqdm(range(5000)) - - for i, _ in enumerate(proc_bar): - - curr_pose = torch.cat([global_aa, composed_aa], dim=1).reshape(1, -1) - mano_output: MANOOutput = mano_layer(curr_pose, zero_shape) - hand_verts_curr = mano_output.verts - - T_g_p = mano_output.transforms_abs # (B, 16, 4, 4) - T_g_a, R, ee = axisIK(T_g_p) - ee_tmplind_ind = ee[:, 1:4] # (B, 3, 3) - - b_loss = torch.abs(ee_tmplind_ind[:, :, 0]).mean() - u_loss = torch.abs(ee_tmplind_ind[:, :, 1]).mean() - l_loss = torch.abs(ee_tmplind_ind[:, :, 2]).mean() - - loss = b_loss + u_loss + l_loss - proc_bar.set_description(f"b: {b_loss.item():.5f} | " - f"u: {u_loss.item():.5f} | " - f"l: {l_loss.item():.5f} | ") - loss.backward() - optimizer.step() - scheduler.step() - - # ===== draw hand curr >>>>> - if i % 10 == 0: - if geometry_to_viz["coord_system_list"] is not None: - viz_ctx.remove_geometry_list(geometry_to_viz["coord_system_list"]) - - coord_system_list = [] - for i in range(T_g_a.shape[1]): - coord_system_list += create_coord_system_can(scale=1, transf=T_g_a[0, i].detach().cpu().numpy()) - viz_ctx.add_geometry_list(coord_system_list) - geometry_to_viz["coord_system_list"] = coord_system_list - - if geometry_to_viz.get('hand_mesh_curr', None) is None: - o3d_hand_mesh_curr = o3d.geometry.TriangleMesh() - o3d_hand_mesh_curr.triangles = o3d.utility.Vector3iVector(hand_faces.detach().cpu().numpy()) - o3d_hand_mesh_curr.vertices = o3d.utility.Vector3dVector(hand_verts_curr[0].detach().cpu().numpy()) - o3d_hand_mesh_curr.compute_vertex_normals() - o3d_hand_mesh_curr.compute_triangle_normals() - o3d_hand_mesh_curr.paint_uniform_color([0.9, 0.0, 0.0]) - viz_ctx.add_geometry(o3d_hand_mesh_curr) - geometry_to_viz["hand_mesh_curr"] = o3d_hand_mesh_curr - else: - o3d_hand_mesh_curr = geometry_to_viz["hand_mesh_curr"] - o3d_hand_mesh_curr.vertices = o3d.utility.Vector3dVector(hand_verts_curr[0].detach().cpu().numpy()) - o3d_hand_mesh_curr.compute_vertex_normals() - o3d_hand_mesh_curr.compute_triangle_normals() - viz_ctx.update_geometry(o3d_hand_mesh_curr) - # <<<<< - - viz_ctx.step() - - -if __name__ == "__main__": - main() diff --git a/doc/pose_correction.gif b/doc/pose_correction.gif new file mode 100644 index 0000000000000000000000000000000000000000..92723b866a78de019beedb39c77df9807a05628a GIT binary patch literal 3113648 zcmV(|K+(TPNk%w1VYmVb0`~v_|Ns9rD0ojulmGz1006Q804oa%dwjc8uCA-a#lpeC zzrDS?ySuruv9H6w0kEwB*w6^SyEcl4HQ?QBqM%Qvq;HasSCx})n3jXKw4&A2&7z={ z)zr?;&BxQfT-({tzrMM)wXo69%gf5e$j8FP#J|D7yY=+%>+0p@cx!0s+F^$ufn*mI9#Ap+>;w~1YzUP(Q{n@Zv7nCJmg~ksmPoC*J2ES0
zrup5Q=goPxF+1S7XO@Uod~wDbSMw>Pk)~>CADV93>Zn;{*cHyJhWf9+uzo8?u1klO
zmot9c=nU4&Ap2~y%P1qYowp5j2!inYHl1Yh_T6saBYQd01P$-oZ+5% W*pb_1}r33EUZawe1ipI
z;Hc=J>S5P=a0?@b|AZqpLKyTI7!W}}=!E|`5)n*TZE%1FuvqA+LYYYgHIU*CLP7_i
z;!UN(1_0qI>e=&2MD)etM$IBELfE{06cdsZiQtMPAOkwp$`t-$6=LBq&chaRAu`S+
zO_qi-G9xp>-%SM*G+te_44yFXMK$){lVqbI03aRKgdOq%II;i_JS7Gg!Laq;Iie#F
z`~WP(4Q*_|=y8Zc0)frZBX_9FJ%&UVIHDkofm*7iKl~%2kk<}eB0;Vli>X*ADkLaI
zh(nG6G~9qh;#4VYz$suCow%Y#!eT}R#}R%cNIt^#ktDU5`59
z1O=BMk3&c&5FNd!*c2LDtBD2E6Nq6smxx+!tcY_t@9UX{ZaZ&IUdZ!3pOilDbHR!6
z4KN2&L
^S
zvp)46)4~K(a0UMXRa-T17Y|lvwL7PfW|XE^_a8NewOEsNeLZ(OHHFH3!wW2pMyTmv
z8NwNmfegd7Tz@xz{KCHsWlXe_|0IZpeslE>2w(^H<;h795O!f7rugD*0Mtz@Sa9rp
zbl@HwF3ZD!eK7-6j9x{6X3KBjj=)TLc77zo;?7hi&I2`1cpaR!h2KGHGXn>(c5Ayf
zpkDK8bnk4>_B!VAZR2)hkh357c5t_N)C@N}Z?$pv%C{)5g7x25`@k_N#ymW?kC(Mx
zPq)gY
XyNmt{kMWU|mMGTw;LN`m
zu;j4QZS1q11o6JF$HAv*6(n0&ja86Ij^?z=B>?4&eU5AHhT82lf}QfSmy+S0I^`4#
zgs=9uZ{23&C8X@TH#SQYCSQ7=awF%sQ1(K8Pt{(=Ka+(Ei8n+%Pq-N
zx++H^P@zm_6RwVc?1m^vOxCZbCvf?jmxGz_xrR^!%ICJqZ{ANQ9Qr}v8WEgXM6X@s
z*>9O?U}t2E*qTrvyO_c+mOpczQwLp0N>|u|gSWd(#F=c4|Eo})OnZkvOGz7sDl$r2
z0-nk$&+V=_u_=$q3&>&x_{7Y|7Axk~&d= Rt{iwlXtIF?U4rpfiDcP^V5HWK!S
zCDwjC%Y)04mlAGa__0`{*B3^dZM5D*Oj~2?&2)Xu+B_~TXCQD5w%UmYQLnbRyCM!!Oe>G254Nq1tPu3hw);>