Code and reproducibility
Implementation, experiments and models Uncompressed size
clear/
assets/
g1/
meshes/
- head_link.STL
- left_ankle_pitch_link.STL
- left_ankle_roll_link.STL
- left_elbow_link.STL
- left_hip_pitch_link.STL
- left_hip_roll_link.STL
- left_hip_yaw_link.STL
- left_knee_link.STL
- left_shoulder_pitch_link.STL
- left_shoulder_roll_link.STL
- left_shoulder_yaw_link.STL
- left_wrist_pitch_link.STL
- left_wrist_roll_link.STL
- left_wrist_yaw_link.STL
- logo_link.STL
- pelvis.STL
- pelvis_contour_link.STL
- right_ankle_pitch_link.STL
- right_ankle_roll_link.STL
- right_elbow_link.STL
- right_hip_pitch_link.STL
- right_hip_roll_link.STL
- right_hip_yaw_link.STL
- right_knee_link.STL
- right_shoulder_pitch_link.STL
- right_shoulder_roll_link.STL
- right_shoulder_yaw_link.STL
- right_wrist_pitch_link.STL
- right_wrist_roll_link.STL
- right_wrist_yaw_link.STL
- torso_link_rev_1_0.STL
- waist_roll_link_rev_1_0.STL
- waist_yaw_link_rev_1_0.STL
- g1.xml
- g1_pos.xml
- g1_pos_native.xml
- README_upstream.md
- scene_flat.xml
- scene_flat_native.xml
- scene_push.xml
- scene_push_native.xml
husky/
meshes/
- base_link.stl
- bumper.stl
- top_plate.stl
- user_rail.stl
- wheel.stl
- husky_wo_plate.xml
- README.md
- scene_husky_wo_plate.xml
spot/
meshes/
- arm_link_el0.obj
- arm_link_el0_coll.obj
- arm_link_el1_0.obj
- arm_link_el1_1.obj
- arm_link_el1_lip.obj
- arm_link_el1_main.obj
- arm_link_fngr_0.obj
- arm_link_fngr_1.obj
- arm_link_hr0.obj
- arm_link_hr0_coll.obj
- arm_link_sh0.obj
- arm_link_sh0_base.obj
- arm_link_sh0_left_motor.obj
- arm_link_sh0_right_motor.obj
- arm_link_sh1_0.obj
- arm_link_sh1_1.obj
- arm_link_wr0.obj
- arm_link_wr0_0.obj
- arm_link_wr0_1.obj
- arm_link_wr1.obj
- arm_link_wr1_0.obj
- arm_link_wr1_1.obj
- body_0.obj
- body_1.obj
- body_collision.obj
- front_jaw.obj
- front_left_hip.obj
- front_left_lower_leg.obj
- front_left_upper_leg_0.obj
- front_left_upper_leg_1.obj
- front_right_hip.obj
- front_right_lower_leg.obj
- front_right_upper_leg_0.obj
- front_right_upper_leg_1.obj
- jaw_tooth.obj
- left_finger.obj
- left_hinge.obj
- left_lower_leg_collision.obj
- left_tooth.obj
- left_upper_leg_collision.obj
- middle_jaw.obj
- rear_left_hip.obj
- rear_left_lower_leg.obj
- rear_left_upper_leg_0.obj
- rear_left_upper_leg_1.obj
- rear_right_hip.obj
- rear_right_lower_leg.obj
- rear_right_upper_leg_0.obj
- rear_right_upper_leg_1.obj
- right_finger.obj
- right_hinge.obj
- right_lower_leg_collision.obj
- right_tooth.obj
- right_upper_leg_collision.obj
- LICENSE
- README.md
- scene.xml
- spot.png
- spot.xml
spot_w_arm/
- scene_flat_native.xml
- scene_interlimb.xml
- scene_namo.xml
- scene_push.xml
- spot_arm.xml
- spot_arm_pos.xml
- spot_arm_pos_native.xml
- a_grid.png
- b_ordered_manipulation.png
- c_maze_navigation.png
- d_long_horizon_planning.png
- teaser.png
ckpts/
- grid/
- locomotion/
- maze/
- paths.json
clear/
data/
- __init__.py
- capability.py
- data.py
- grid.py
- kappa.py
- modes.py
- morphology.py
- oracle.py
- probes.py
- terrain.py
maze/
- __init__.py
- robots.py
- world.py
model/
- __init__.py
- affordancenet.py
- causalflownet.py
- clear.py
- encoders.py
- morphology_encoder.py
- ordernet.py
- quantizers.py
- QUANTIZERS_LICENSE
- __init__.py
- eval.py
- paths.py
- provenance.py
- util.py
experiments/
common/
- __init__.py
- dispatch.py
- interfaces.py
- manifests.py
- registry.py
- statistics.py
exp1/
configs/
- debug.yaml
- leap2026_plancode.yaml
- leap2026_rankmask_causal.yaml
- leap2026_rankmask_nocausal.yaml
- main.yaml
- paper.json
data/
ordering_main/
test/
- scene_manifest.jsonl
- scene_manifest.jsonl.sha256
train/
- scene_manifest.jsonl
val/
- scene_manifest.jsonl
- generation_stats.json
- templates.json
- original_fresh.jsonl
- original_remaining.jsonl
- structural.jsonl
- __init__.py
- abstraction.py
- aggregate.py
- cli.py
- coverage_sampling.py
- dataset.py
- environment.py
- evaluate.py
- evaluator.py
- generator.py
- joint_matching_model.py
- metrics.py
- models.py
- oracle.py
- ordering_ablation.py
- plancode_model.py
- planslots_model.py
- planslots_scene_model.py
- planslots_state_model.py
- reachability_study.py
- render.py
- sampling_dev.py
- setcover_model.py
- setcover_v2_model.py
- shared_model.py
- structural_confirmation.py
- structural_stress.py
- structural_training.py
exp2/
data/paper/
- collection_audit.json
- collection_protocol.json
- train.npz
- val.npz
- __init__.py
- access_dataset.py
- access_teacher.py
- binding.py
- comparison_models.py
- comparison_protocol.py
- dataset.py
- evaluate_full.py
- evaluation_manifest.py
- external_models.py
- model.py
- refine_flow_comparison.py
- run_scene.py
- scene_bridge.py
- splits.py
- sumo_executor.py
- target_codec.py
- train_comparison.py
exp3/
data/
primary_prepared_local_edges/
- boundary_attempt_audit.json
- g1_probes.jsonl
- g1_teachers.jsonl
- manifest.json
- shared_probes.jsonl
- shared_teachers.jsonl
- spot_arm_probes.jsonl
- spot_arm_teachers.jsonl
probe_scenes/
16e3fa0c0c5c38661a773b350944f17959e24302fcdc0ff56c052a1f2be98795/
- input.jsonl
277fb2f20cac8283a7349ccb5586178d01f8d896713503ff2990c4183f04f34e/
- input.jsonl
41e15b45fdccc79250b48d52490b266341533e2a647d8caf3ac7a9ba8f0b4148/
- input.jsonl
5b2f41b53e92a384b2e1250f85e41c4f09ba082af38d50462629081b1ddf431d/
- input.jsonl
651a14b652c3e3b11f7fa159c157f2c76676a3b9e3fc89287eaf3e9da61d6616/
- input.jsonl
70ad2fa06f1e544656b78007648b42e769e228c45b158897b0897bdb9e2409bc/
- input.jsonl
70d3ff9e6a4d9c791040683d8e5c0b325f23dae67c4bd613554c97db0bb059d0/
- input.jsonl
782631f2b2a894ad2de71e44d38c210e0786d14701a150ce9b543cc0c8863be5/
- input.jsonl
c61f1ff7382b71f7874fae054ef5c0ac742b100caa8a1b594e0d0e2d0cbc6c95/
- input.jsonl
ce5879da6d701f3e5f5acf1b0157f355f063388218a54e0ef7a6b88200e91dd0/
- input.jsonl
d611aa2a4841d0339322e7a6bc63dedc6f6c6d9982938945b937aa42d728e5a2/
- input.jsonl
dbd2a35f769ff1be8840faf254d45bae891f6018aae6edd3da76a13a930a57b5/
- input.jsonl
e8841fcc7ad0f57c24a54eff6fea2cd6fe51db80e37f1d56463d44a0e461af30/
- input.jsonl
- example_g1.json
trajectory_v3/
- __init__.py
- calibration.py
- cdgs.py
- cli.py
- contracts.py
- data.py
- dataset_gen.py
- evaluate_shared.py
- evaluate_v3.py
- exp3_protocol.py
- get_encoder.py
- model.py
- navigation.py
- navigation_cache.py
- navigation_planner.py
- navigation_scenes.py
- oracle_capability.py
- parallel_shared_eval.py
- plan_audit.py
- review_metrics.py
- reviewed_task_backend.py
- robot_id_encoder.py
- scenes.py
- shared_cli.py
- shared_morphology.py
- summarize_shared.py
- summarize_shared_exp4.py
- task_backend.py
- training_state.py
- traversability.py
- traversal_probes.py
- unseen_bodies.py
- __init__.py
- attribution.py
- model.py
- oracle.py
- scene_io.py
- surface_adapter.py
- train.py
exp4/
sprint6h/
- train_adapt.py
- __init__.py
- __init__.py
models/baselines/
autoregressive_order/
- __init__.py
- model.py
gumbel_sinkhorn_order/
- __init__.py
- model.py
- ops.py
polynet_order/
- __init__.py
- model.py
- __init__.py
utils/
tests/
- __init__.py
- conftest.py
- test_candidate_orders_guard.py
- test_flow_trace.py
- test_inference_recipe.py
- test_ordering_ablation.py
- test_shared_balanced_sampling.py
- test_shared_morphology_v3.py
tools/
- fetch_checkpoints.py
- get.py
- link_checkpoints.py
- render_plan.py
- run.py
training/
exp1/
stages/
structural_gentle_20260913_d/
- design.json
- train.jsonl
structural_replay_20260913_c/
- design.json
- train.jsonl
structural_training_20260913_a/
- design.json
- train.jsonl
validation/structural_stress_20260913_a/
- scene_manifest.jsonl
- ancestry.json
exp2/
- train_defaults.json
exp3/configs/
seed0/
- cdgs.json
- clear.json
- clear_holdout_spotarm.json
- clear_nocausal.json
- get_encoder.json
- get_encoder_holdout_spotarm.json
- no_affordance.json
- noemb.json
- noemb_holdout_spotarm.json
- robot_id.json
- robot_id_holdout_spotarm.json
- specialist_g1.json
- specialist_spot_arm.json
seed1/
- cdgs.json
- clear.json
- clear_holdout_spotarm.json
- clear_nocausal.json
- get_encoder.json
- get_encoder_holdout_spotarm.json
- no_affordance.json
- noemb.json
- noemb_holdout_spotarm.json
- robot_id.json
- robot_id_holdout_spotarm.json
- specialist_g1.json
- specialist_spot_arm.json
seed2/
- cdgs.json
- clear.json
- clear_holdout_spotarm.json
- clear_nocausal.json
- get_encoder.json
- get_encoder_holdout_spotarm.json
- no_affordance.json
- noemb.json
- noemb_holdout_spotarm.json
- robot_id.json
- robot_id_holdout_spotarm.json
- specialist_g1.json
- specialist_spot_arm.json
exp4/
data/
- adapt_probes.jsonl
- adapt_teachers.jsonl
- exp4_teachers.jsonl
- exp4_teachers_summary.json
- config.json
viz/
exp3/
- __init__.py
- render.py
model/
- __init__.py
- style.py
- viz.py
- __init__.py
- asset
- pyproject.toml
- README.md
- run_01_grid.py
- run_02_manipulation.py
- run_03_maze.py
- run_04_replanning.py
Run the planner
uv · Python 3.11 · CPUcurl -LO https://anonymous-hf.com/api/a/p5icltdspejh/resolve/utils/tools/get.py
python3 get.py clear && cd clear
uv sync
uv run run_03_maze.py plan --input experiments/exp3/data/example_g1.json # window
uv run run_03_maze.py plan --input experiments/exp3/data/example_g1.json --video demo.mp4 # headless mp4
uv run run_01_grid.py evaluate --method clear --out outputs/grid/clear # or a baseline: polynet, ...Implementation
01
Representation
Robot structure is encoded together with object and scene features.
Visualization
G1
Spot
Spot + arm
Husky
Object states
Stairs
Stepped bridge
Slope
Mixed terrain
Code
SharedEncoder
class SharedEncoder(nn.Module): def __init__( self, dim, object_dim, scene_dim, structure_dim, depth=4, heads=4, dim_head=None, navigation_dim=0, navigation_norm=False, ): super().__init__() dim_head = dim_head or dim // heads self.object_in = _mlp(object_dim, dim) self.structure_in = StructureEncoder(structure_dim, dim) self.scene_in = _mlp(scene_dim, dim) if scene_dim else None if not navigation_dim: self.navigation_in = None elif navigation_norm: self.navigation_in = NormalizedNavigation(navigation_dim, dim) else: self.navigation_in = nn.Linear(navigation_dim, dim, bias=False) self.trunk = AffordanceTransformer( dim, depth=depth, heads=heads, dim_head=dim_head ) def forward(self, batch, no_embodiment=False): objects = self.object_in(batch["object_features"]) if self.navigation_in is not None: navigation = batch["navigation_features"] if no_embodiment: navigation = torch.zeros_like(navigation) objects = objects + self.navigation_in(navigation) embodiment = self.structure_in.pool( batch["structure"], batch["structure_mask"] )[:, None] if no_embodiment: embodiment = torch.zeros_like(embodiment) scene = ( self.scene_in(batch["scene_features"]) if self.scene_in is not None else batch["scene_tokens"] ) b, n, _ = objects.shape ones = torch.ones(b, 1, dtype=torch.bool, device=objects.device) scene_mask = batch.get("scene_mask") if scene_mask is None: scene_mask = torch.ones( b, scene.shape[1], dtype=torch.bool, device=objects.device ) key_mask = torch.cat([ones, batch["object_mask"], scene_mask], 1) tokens = torch.cat([embodiment, objects, scene], 1) blocked = (~key_mask)[:, None, :].expand(-1, key_mask.shape[1], -1) encoded = self.trunk(tokens, mask=blocked) return dict( tokens=encoded, objects=encoded[:, 1 : 1 + n], scene_tokens=encoded[:, 1 + n :], embodiment=embodiment, scene=scene, scene_mask=scene_mask, key_mask=key_mask, )02
Selection and order
Object selection and latent interaction priorities.
Visualization
Code
OrderingHead
class OrderingHead(nn.Module): """Row 0 of `priority` is the selection logit, rows 1-2 are (mu, raw sigma).""" def __init__(self, dim, sigma_min=0.05, sigma_max=0.5): super().__init__() self.priority = nn.Linear(dim, 3) self.sigma_min, self.sigma_max = sigma_min, sigma_max def forward(self, objects): selection, mu, raw = self.priority(objects).unbind(-1) return selection, mu, priority_scale(raw, self.sigma_min, self.sigma_max) def loss( self, selection, mu, sigma, rank, object_mask, generator=None, rank_selected_only=False, ): n = rank.shape[1] selected = object_mask & (rank >= 0) & (rank < n) if not object_mask.any(): zero = _zero_like_loss(selection, mu, sigma) return dict(selection=zero, order=zero, kl=zero) selection_loss = F.binary_cross_entropy_with_logits( selection[object_mask], selected[object_mask].float() ) u = mu + sigma * torch.randn(mu.shape, device=mu.device, generator=generator) sentinel = torch.where(selected, rank, torch.full_like(rank, n)) # Alternative valid subsets can contradict selected-vs-unselected priority pairs. order = sampled_priority_rank_loss( u, sentinel, selected if rank_selected_only else object_mask ) kl = 0.5 * (mu**2 + sigma**2 - 1) - sigma.log() return dict(selection=selection_loss, order=order, kl=kl[object_mask].mean()) def sample( self, selection, mu, sigma, object_mask, generator=None, max_interactions=None, sample_selection=False, ): if sample_selection: draw = torch.rand( selection.shape, device=selection.device, generator=generator ) selected = (draw < torch.sigmoid(selection)) & object_mask else: selected = (selection > 0) & object_mask u = mu + sigma * torch.randn(mu.shape, device=mu.device, generator=generator) rank, active = rank_selected(u, selected, max_interactions) return rank.masked_fill(~active, -1), active03
Trajectory generation
Object trajectory generation conditioned on selection and order.
Visualization
Code
RankCausalTargetFlow
class RankCausalTargetFlow(nn.Module): def __init__( self, dim, target_dim, depth=4, heads=4, dim_head=32, causal=True, causal_bias_slope=0.0, ): super().__init__() if dim % 4: raise ValueError("target flow dim must be divisible by 4") self.dim, self.target_dim, self.causal = dim, target_dim, causal self.noise_std = 1.0 self.causal_bias_slope = float( causal_bias_slope ) # soft-causal ablation, see causal_velocity_field.causal_bias self.cfg_in = nn.Sequential( nn.Linear(dim + target_dim + dim // 2, dim), nn.GELU(), nn.Linear(dim, dim) ) self.trunk = AffordanceTransformer( dim, depth=depth, heads=heads, dim_head=dim_head ) self.head = nn.Sequential( nn.LayerNorm(dim), nn.Linear(dim, dim), nn.GELU(), nn.Linear(dim, target_dim), ) def forward(self, static, context, scene, g_t, t, rank, selected, key_mask=None): return target_velocity( static, context, scene, g_t, t, rank, selected, self.cfg_in, self.trunk, self.head, self.dim // 2, self.causal, key_mask=key_mask, causal_bias_slope=self.causal_bias_slope, ) @torch.no_grad() def sample( self, static, context, scene, rank, selected, n_steps=30, generator=None, key_mask=None, trace=None, ): """`trace`, if a list, receives (t, state) at every Euler step and at t=1. The states are the generator's own intermediates. They exist so a figure can show the flow evolving instead of an interpolation of its endpoint; nothing in the planner reads them and the default path is unchanged. """ if n_steps <= 0: raise ValueError("n_steps must be positive") rank, selected = selected_ranks(rank, selected) g = ( torch.randn( *static.shape[:2], self.target_dim, device=static.device, dtype=static.dtype, generator=generator ) * self.noise_std ) g = torch.where(selected[..., None], g, torch.zeros_like(g)) for k in range(n_steps): t = torch.full( (static.shape[0],), k / n_steps, device=static.device, dtype=static.dtype, ) if trace is not None: trace.append((k / n_steps, g.detach().clone())) v = self(static, context, scene, g, t, rank, selected, key_mask=key_mask) g = g + torch.where(selected[..., None], v, torch.zeros_like(v)) / n_steps if trace is not None: trace.append((1.0, g.detach().clone())) return g04
Feasibility checking
Sequential validation accounts for scene changes induced by preceding interactions.
Visualization
Code
validate_sequence
def validate_sequence(scene, embodiment, plan, navigator, assessor): work = deepcopy(scene) steps = [] for path in plan.paths: obj = next( (o for o in work["objects"] if o["object_id"] == path.object_id), None ) if obj is None: return None, "UNKNOWN_OBJECT" if ( np.linalg.norm(np.asarray(path.start[:2]) - obj["pose"][:2]) > 0.08 or abs(angle_delta(path.start[2], obj["pose"][2])) > 0.12 ): return None, "PATH_START_MISMATCH" if path_collision(work, path): return None, "OBJECT_PATH_COLLISION" navigation = navigator.analyze(work, embodiment) assessment = assessor.assess(work, embodiment, path, navigation) if assessment.status != "candidate_feasible": return None, "INTERACTION_" + assessment.status.upper() approach = navigation.route_to(assessment.approach_xy) if approach is None: return None, "APPROACH_ROUTE_NOT_FOUND" obj["pose"] = list(path.goal) work["start"] = [*assessment.predicted_robot_xy_after, 0.0] steps.append(dict(path=path, approach=approach, assessment=assessment)) navigation = navigator.analyze(work, embodiment) if navigation.goal_route is None: return None, "POST_INTERACTION_GOAL_ROUTE_NOT_FOUND" return dict(steps=steps, route=navigation.goal_route), None05
Execution
G1 and Spot + arm track object references using updated state estimates.
Visualization
Ego RGB
Ego RGB
Code
AbstractController.interact
def interact( self, object_id, target_configuration, contact_point=None, body_part=None ) -> ExecutionResult: """Move `object_id` to the target cell with `body_part`. Preconditions (all checked with this controller's feasibility model): the part exists and can move the object, a Sokoban route exists, and every standing cell along it is reachable by the body. On success the object is at the target and the robot where the last move left it. """ before_obj, before_rob = self.object_state(), self.robot_state() try: obj = self.scene.object_by_id(int(object_id)) except KeyError: return self._result( False, FailureType.WRONG_OBJECT_SELECTION, before_obj, before_rob, {"reason": f"unknown object {object_id}"}, ) self.attempted.add(int(object_id)) target = tuple( int(v) for v in ( target_configuration if len(target_configuration) == 2 else world_to_cell(target_configuration, self.scene.cell_m) ) ) part = self.body.part(body_part) if body_part is not None else None if part is None: return self._result( False, FailureType.CONTACT_FAILURE, before_obj, before_rob, {"reason": f"body part {body_part!r} unavailable"}, ) ok, why = self.body.part_can_move(part, obj) if not ok: return self._result( False, FailureType(why), before_obj, before_rob, {"reason": f"{part.name} cannot move object {object_id}", "check": why}, ) if target == tuple(obj.cell): return self._result( True, None, before_obj, before_rob, {"reason": "already at target"} ) if ( not self.scene.in_bounds(target) or self.scene.is_wall(target) or target in self.scene.object_blocked_cells(obj) or any(o is not obj and tuple(o.cell) == target for o in self.scene.objects) ): return self._result( False, FailureType.UNREACHABLE_TARGET, before_obj, before_rob, {"reason": "target cell not placeable", "target": list(target)}, ) route = object_route( self.scene, obj, target, can_drag=bool(part.can_grasp), robot_cell=self.robot, ) if route is None: return self._result( False, FailureType.UNREACHABLE_TARGET, before_obj, before_rob, {"reason": "no push/pull route", "target": list(target)}, ) cells, modes = route others = {tuple(o.cell) for o in self.scene.objects if o is not obj} for k in range(len(cells) - 1): a, b = cells[k], cells[k + 1] d = (b[0] - a[0], b[1] - a[1]) if modes[k] == "push": cands = [ s for s in standing_cells(a, d) if self.scene.in_bounds(s) and s not in others and s != a and not self.scene.is_wall(s) ] legs = [ ( s, bfs_cells( self.scene, self.body, self.robot, s, blocked=others | {a} ), ) for s in cands ] legs = [(s, p) for s, p in legs if p is not None] if not legs: return self._result( False, FailureType.UNREACHABLE_TARGET, before_obj, before_rob, { "reason": "standing cell unreachable", "object_cell": list(a), "dir": list(d), }, ) s, p = min(legs, key=lambda t: len(t[1])) self.trajectory += p[1:] self.legs.append([list(c) for c in p]) obj.cell = b # the pusher advances into the cell the object left (if it can enter it) if ( self.scene.can_enter(self.body, s, a) and abs(s[0] - a[0]) + abs(s[1] - a[1]) == 1 ): self.robot = a self.trajectory.append(a) else: self.robot = s else: # pull: robot stands on the destination, grasps, retreats one cell further p = bfs_cells( self.scene, self.body, self.robot, b, blocked=others | {a} ) retreat = (b[0] + d[0], b[1] + d[1]) if ( p is None or not self.scene.in_bounds(retreat) or retreat in others or not self.scene.can_enter(self.body, b, retreat) ): return self._result( False, FailureType.UNREACHABLE_TARGET, before_obj, before_rob, {"reason": "pull stance unreachable", "object_cell": list(a)}, ) self.trajectory += p[1:] self.legs.append([list(c) for c in p]) obj.cell = b self.robot = retreat self.trajectory.append(retreat) return self._result( True, None, before_obj, before_rob, { "route": [list(c) for c in cells], "modes": list(modes), "part": part.name, }, )