{ "export_format": "kindly-labeling-export/1.0", "schema_version": "0.1.1", "vocab_ref": "public:cholect50.triplet (ivtmetrics maps.txt) + cholec80.phases", "synthetic": false, "_readme": { "what_this_is": "Kindly's FIRST non-synthetic worked episode. It maps the PUBLISHED ground-truth annotation taxonomy of two public laparoscopic-cholecystectomy datasets — Cholec80 (7 surgical phases + 7-tool presence) and CholecT50 (100 action-triplet classes) — into the Kindly four-layer clinical-episode schema, to demonstrate the schema interoperates with REAL surgical annotations, not just synthetic paper examples.", "honesty": "synthetic:false because the taxonomy, phase set, tool set, and triplet classes are all REAL values from real public datasets (cited in episode_envelope.source), not invented. This is a MAPPING DEMONSTRATION, not a redistribution of the datasets: NO video and NO restricted per-frame ground-truth annotation files are reproduced here. Per-frame timestamps/durations are REPRESENTATIVE placements illustrating the documented phase/triplet structure of a cholecystectomy — they are NOT copied frame-level ground truth (which is gated under the dataset license and was not accessed). See episode_envelope.source for full attribution, license, and provenance.", "layers": "L1 = Cholec80 phase taxonomy + documented order (REAL). L2 = CholecT50 triplet classes (REAL; each carries its ivtmetrics class id). L3 = Cholec80 7-tool presence, mapped to a coarse in_field/absent zone (REAL tool set). L4 = INTENTIONALLY EMPTY — the public corpora do not annotate workflow-context (interruptions/rework/deviations/handoffs), which is exactly the layer Kindly's schema adds.", "not_a_claim": "Kindly Robotics did NOT capture, own, or license this footage, and has NO affiliation with the dataset authors (CAMMA, University of Strasbourg / IHU Strasbourg). Use of the public datasets is governed solely by their CC BY-NC-SA 4.0 license.", "structure": "episode_envelope -> labels.L1 (phase/step) -> labels.L2 (action triplets) -> labels.L3 (instrument-presence tracks) -> labels.L4 (empty) -> rlds_excerpt (a derived RLDS slice). The sidecar (L1-L4) is the source of truth; the RLDS view is a regenerable flattening." }, "episode_envelope": { "episode_id": "kr-cholect50-cholecystectomy-demo-0001", "schema_version": "0.1.1", "synthetic": false, "data_provenance": "public_dataset_derived", "workflow_family": "OR.laparoscopic_cholecystectomy", "boundary_def": "OR.laparoscopic_cholecystectomy@cholec80-phase-progression (start = preparation; end = gallbladder retraction)", "vocab_ref": "public:cholect50.triplet (ivtmetrics maps.txt, BSD-2-Clause) + cholec80.phases + cholec80.tool_presence", "program_phase": "public_dataset_mapping", "capture": { "fps": 25, "duration_s": 2100.0, "modalities": ["rgb_laparoscopic"], "frame_count": 52500 }, "deid": { "status": "verified", "method": "source_dataset_public_release (de-identified by the dataset authors prior to public release; internal endoscopic view, no identifiable patient features)", "phi_residual_risk": "low" }, "consent": { "patient_in_frame": false, "note": "Public research dataset (Cholec80 / CholecT50), released de-identified by its authors under CC BY-NC-SA 4.0. Kindly obtained no separate consent and captured nothing; use is governed solely by the dataset license." }, "task_summary": "Laparoscopic cholecystectomy (gallbladder removal), as annotated by the public Cholec80 (7-phase workflow + 7-tool presence) and CholecT50 ( action triplets) datasets, mapped into the Kindly four-layer schema. Workflow/annotation-structure level; representative timings.", "annotation_provenance": { "annotators": ["source: CAMMA Cholec80 phase + tool-presence ground truth", "source: CAMMA CholecT50 action-triplet ground truth"], "clinical_reviewer": "source: dataset published under clinical annotation by CAMMA / IHU Strasbourg (not re-reviewed by Kindly)", "consensus_method": "mapped_from_public_dataset (Kindly remapped the datasets' PUBLISHED label taxonomy + documented phase progression into the schema layers; see episode_envelope.source)" }, "source": { "type": "public_dataset_mapping_demonstration", "datasets": [ { "name": "CholecT50", "full_name": "CholecT50 surgical action triplet dataset", "provider": "CAMMA, University of Strasbourg / IHU Strasbourg", "url": "https://github.com/CAMMA-public/cholect50", "license": "CC BY-NC-SA 4.0", "license_url": "https://creativecommons.org/licenses/by-nc-sa/4.0/", "used_for": "L2 action triplets: the taxonomy (6 instruments, 10 verbs, 15 targets) and its 100 triplet classes.", "citation": "C.I. Nwoye, T. Yu, C. Gonzalez, B. Seeliger, P. Mascagni, D. Mutter, J. Marescaux, N. Padoy. Rendezvous: Attention Mechanisms for the Recognition of Surgical Action Triplets in Endoscopic Videos. Medical Image Analysis 78:102433, 2022." }, { "name": "Cholec80", "full_name": "Cholec80 laparoscopic cholecystectomy dataset", "provider": "CAMMA, University of Strasbourg", "url": "https://camma.unistra.fr/datasets/", "license": "CC BY-NC-SA 4.0", "license_url": "https://creativecommons.org/licenses/by-nc-sa/4.0/", "used_for": "L1 phase taxonomy (7 phases) + documented phase order; L3 tool-presence taxonomy (7 tools).", "citation": "A.P. Twinanda, S. Shehata, D. Mutter, J. Marescaux, M. de Mathelin, N. Padoy. EndoNet: A Deep Architecture for Recognition Tasks on Laparoscopic Videos. IEEE Transactions on Medical Imaging 36(1):86-97, 2017." }, { "name": "ivtmetrics", "full_name": "ivtmetrics (CholecT50 evaluation library)", "provider": "CAMMA", "url": "https://github.com/CAMMA-public/ivtmetrics", "license": "BSD-2-Clause", "used_for": "triplet -> (instrument, verb, target) class-id map (maps.txt) used to resolve each L2 triplet's ivtmetrics class id." } ], "clip_reference": "Laparoscopic cholecystectomy from the Cholec80 corpus (illustrative case reference: CholecT50 video VID01). The Cholec80 corpus averages ~39 min/case at 25 fps.", "kindly_role": "Kindly Robotics did NOT capture, own, or license this footage and has NO affiliation with CAMMA or the dataset authors. This episode maps the datasets' PUBLISHED annotation taxonomy and the documented cholecystectomy phase progression into the Kindly four-layer schema to demonstrate schema interop on real surgical annotations.", "content_reproduced": "Published label taxonomy (phase / instrument / verb / target names), the BSD-2-Clause triplet->class-id map, and the documented phase order. NO video and NO restricted per-frame ground-truth annotation files are reproduced or redistributed here.", "timing_provenance": "Per-frame timestamps and durations are REPRESENTATIVE placements illustrating the documented phase/triplet structure of a laparoscopic cholecystectomy; they are NOT copied frame-level ground truth (which is gated under the dataset license and was not accessed).", "layers_from_real_labels": "L1 (Cholec80 phase set + order), L2 (CholecT50 triplet classes), L3 (Cholec80 tool-presence set). L4 is intentionally EMPTY: the public corpora do not annotate workflow-context (interruptions / rework / deviations / handoffs) — the layer Kindly adds." } }, "labels": { "L1": [ { "layer": "L1", "t_start_s": 0.0, "t_end_s": 180.0, "phase": "preparation", "step": "preparation", "confidence": "medium", "note": "Cholec80 phase 1/7. NOTE: Cholec80 annotates at the PHASE level only (no published sub-step layer), so L1.step mirrors L1.phase for this source. Phase order is real/documented; boundaries are representative timings (see envelope.source.timing_provenance)." }, { "layer": "L1", "t_start_s": 180.0, "t_end_s": 780.0, "phase": "calot_triangle_dissection", "step": "calot_triangle_dissection", "confidence": "medium", "note": "Cholec80 phase 2/7 — dissection of the hepatocystic (Calot's) triangle to expose the cystic duct and artery. Typically the longest phase." }, { "layer": "L1", "t_start_s": 780.0, "t_end_s": 960.0, "phase": "clipping_and_cutting", "step": "clipping_and_cutting", "confidence": "medium", "note": "Cholec80 phase 3/7 — clip and divide the cystic artery and cystic duct." }, { "layer": "L1", "t_start_s": 960.0, "t_end_s": 1500.0, "phase": "gallbladder_dissection", "step": "gallbladder_dissection", "confidence": "medium", "note": "Cholec80 phase 4/7 — dissect the gallbladder off the liver bed." }, { "layer": "L1", "t_start_s": 1500.0, "t_end_s": 1620.0, "phase": "gallbladder_packaging", "step": "gallbladder_packaging", "confidence": "medium", "note": "Cholec80 phase 5/7 — place the gallbladder into a specimen bag." }, { "layer": "L1", "t_start_s": 1620.0, "t_end_s": 1920.0, "phase": "cleaning_and_coagulation", "step": "cleaning_and_coagulation", "confidence": "medium", "note": "Cholec80 phase 6/7 — irrigation, aspiration, and hemostasis of the liver bed / cavity." }, { "layer": "L1", "t_start_s": 1920.0, "t_end_s": 2100.0, "phase": "gallbladder_retraction", "step": "gallbladder_retraction", "confidence": "medium", "note": "Cholec80 phase 7/7 — retrieve the bagged specimen through the port." } ], "L2": [ { "layer": "L2", "actor_role": "surgeon", "verb": "retract", "object": "omentum", "instrument": "grasper", "triplet_id": "ivt-20", "t_start_s": 60.0, "t_end_s": 95.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-20)." }, { "layer": "L2", "actor_role": "surgeon", "verb": "grasp", "object": "gallbladder", "instrument": "grasper", "triplet_id": "ivt-7", "t_start_s": 95.0, "t_end_s": 140.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-7) — grasp the fundus for exposure." }, { "layer": "L2", "actor_role": "surgeon", "verb": "retract", "object": "gallbladder", "instrument": "grasper", "triplet_id": "ivt-17", "t_start_s": 190.0, "t_end_s": 230.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-17)." }, { "layer": "L2", "actor_role": "surgeon", "verb": "retract", "object": "liver", "instrument": "grasper", "triplet_id": "ivt-19", "t_start_s": 230.0, "t_end_s": 260.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-19)." }, { "layer": "L2", "actor_role": "surgeon", "verb": "dissect", "object": "cystic_plate", "instrument": "hook", "triplet_id": "ivt-59", "t_start_s": 300.0, "t_end_s": 360.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-59)." }, { "layer": "L2", "actor_role": "surgeon", "verb": "dissect", "object": "cystic_duct", "instrument": "hook", "triplet_id": "ivt-58", "t_start_s": 380.0, "t_end_s": 460.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-58)." }, { "layer": "L2", "actor_role": "surgeon", "verb": "dissect", "object": "cystic_artery", "instrument": "hook", "triplet_id": "ivt-57", "t_start_s": 480.0, "t_end_s": 560.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-57)." }, { "layer": "L2", "actor_role": "surgeon", "verb": "dissect", "object": "gallbladder", "instrument": "hook", "triplet_id": "ivt-60", "t_start_s": 600.0, "t_end_s": 660.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-60) — dividing peritoneal attachments." }, { "layer": "L2", "actor_role": "surgeon", "verb": "coagulate", "object": "blood_vessel", "instrument": "bipolar", "triplet_id": "ivt-23", "t_start_s": 680.0, "t_end_s": 720.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-23) — hemostasis." }, { "layer": "L2", "actor_role": "surgeon", "verb": "retract", "object": "gallbladder", "instrument": "grasper", "triplet_id": "ivt-17", "t_start_s": 790.0, "t_end_s": 820.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-17)." }, { "layer": "L2", "actor_role": "surgeon", "verb": "clip", "object": "cystic_artery", "instrument": "clipper", "triplet_id": "ivt-78", "t_start_s": 820.0, "t_end_s": 850.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-78)." }, { "layer": "L2", "actor_role": "surgeon", "verb": "cut", "object": "cystic_artery", "instrument": "scissors", "triplet_id": "ivt-68", "t_start_s": 850.0, "t_end_s": 880.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-68)." }, { "layer": "L2", "actor_role": "surgeon", "verb": "clip", "object": "cystic_duct", "instrument": "clipper", "triplet_id": "ivt-79", "t_start_s": 885.0, "t_end_s": 915.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-79)." }, { "layer": "L2", "actor_role": "surgeon", "verb": "cut", "object": "cystic_duct", "instrument": "scissors", "triplet_id": "ivt-69", "t_start_s": 915.0, "t_end_s": 945.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-69)." }, { "layer": "L2", "actor_role": "surgeon", "verb": "retract", "object": "gallbladder", "instrument": "grasper", "triplet_id": "ivt-17", "t_start_s": 970.0, "t_end_s": 1010.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-17)." }, { "layer": "L2", "actor_role": "surgeon", "verb": "retract", "object": "liver", "instrument": "grasper", "triplet_id": "ivt-19", "t_start_s": 1010.0, "t_end_s": 1050.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-19)." }, { "layer": "L2", "actor_role": "surgeon", "verb": "dissect", "object": "gallbladder", "instrument": "hook", "triplet_id": "ivt-60", "t_start_s": 1080.0, "t_end_s": 1180.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-60) — separating gallbladder from the liver bed." }, { "layer": "L2", "actor_role": "surgeon", "verb": "dissect", "object": "gallbladder", "instrument": "hook", "triplet_id": "ivt-60", "t_start_s": 1220.0, "t_end_s": 1320.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-60) — continued liver-bed dissection." }, { "layer": "L2", "actor_role": "surgeon", "verb": "coagulate", "object": "liver", "instrument": "bipolar", "triplet_id": "ivt-29", "t_start_s": 1360.0, "t_end_s": 1420.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-29) — liver-bed hemostasis." }, { "layer": "L2", "actor_role": "surgeon", "verb": "grasp", "object": "specimen_bag", "instrument": "grasper", "triplet_id": "ivt-12", "t_start_s": 1510.0, "t_end_s": 1545.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-12)." }, { "layer": "L2", "actor_role": "surgeon", "verb": "pack", "object": "gallbladder", "instrument": "grasper", "triplet_id": "ivt-13", "t_start_s": 1545.0, "t_end_s": 1610.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-13) — placing the specimen into the bag." }, { "layer": "L2", "actor_role": "surgeon", "verb": "irrigate", "object": "abdominal_wall_cavity", "instrument": "irrigator", "triplet_id": "ivt-88", "t_start_s": 1630.0, "t_end_s": 1700.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-88)." }, { "layer": "L2", "actor_role": "surgeon", "verb": "aspirate", "object": "fluid", "instrument": "irrigator", "triplet_id": "ivt-82", "t_start_s": 1700.0, "t_end_s": 1760.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-82)." }, { "layer": "L2", "actor_role": "surgeon", "verb": "coagulate", "object": "liver", "instrument": "bipolar", "triplet_id": "ivt-29", "t_start_s": 1780.0, "t_end_s": 1840.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-29)." }, { "layer": "L2", "actor_role": "surgeon", "verb": "irrigate", "object": "liver", "instrument": "irrigator", "triplet_id": "ivt-90", "t_start_s": 1850.0, "t_end_s": 1900.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-90)." }, { "layer": "L2", "actor_role": "surgeon", "verb": "grasp", "object": "specimen_bag", "instrument": "grasper", "triplet_id": "ivt-12", "t_start_s": 1930.0, "t_end_s": 2000.0, "confidence": "medium", "note": "CholecT50 triplet (class ivt-12) — extracting the bagged specimen." } ], "L3": [ { "layer": "L3", "object_id": "obj_grasper", "object_class": "grasper", "state": { "zone": "in_field", "sterility": "sterile", "in_count": false }, "transition_from": null, "t_s": 60.0, "count_event": null, "confidence": "medium", "note": "Cholec80 tool PRESENCE mapped to a coarse in_field/absent zone. sterility=sterile reflects the intraoperative sterile field; in_count is not a concept in this source (set false). Grasper is present across the case for retraction." }, { "layer": "L3", "object_id": "obj_hook", "object_class": "hook", "state": { "zone": "in_field", "sterility": "sterile", "in_count": false }, "transition_from": { "zone": "absent", "sterility": "sterile", "in_count": false }, "t_s": 300.0, "count_event": null, "confidence": "medium", "note": "Hook present during Calot dissection and gallbladder dissection." }, { "layer": "L3", "object_id": "obj_bipolar", "object_class": "bipolar", "state": { "zone": "in_field", "sterility": "sterile", "in_count": false }, "transition_from": { "zone": "absent", "sterility": "sterile", "in_count": false }, "t_s": 680.0, "count_event": null, "confidence": "medium", "note": "Bipolar present for hemostasis (Calot and cleaning/coagulation phases)." }, { "layer": "L3", "object_id": "obj_clipper", "object_class": "clipper", "state": { "zone": "in_field", "sterility": "sterile", "in_count": false }, "transition_from": { "zone": "absent", "sterility": "sterile", "in_count": false }, "t_s": 820.0, "count_event": null, "confidence": "medium", "note": "Clipper present during the clipping-and-cutting phase." }, { "layer": "L3", "object_id": "obj_clipper", "object_class": "clipper", "state": { "zone": "absent", "sterility": "sterile", "in_count": false }, "transition_from": { "zone": "in_field", "sterility": "sterile", "in_count": false }, "t_s": 915.0, "count_event": null, "confidence": "medium", "note": "Clipper leaves the field after clip application." }, { "layer": "L3", "object_id": "obj_scissors", "object_class": "scissors", "state": { "zone": "in_field", "sterility": "sterile", "in_count": false }, "transition_from": { "zone": "absent", "sterility": "sterile", "in_count": false }, "t_s": 850.0, "count_event": null, "confidence": "medium", "note": "Scissors present to divide the clipped cystic artery and duct." }, { "layer": "L3", "object_id": "obj_scissors", "object_class": "scissors", "state": { "zone": "absent", "sterility": "sterile", "in_count": false }, "transition_from": { "zone": "in_field", "sterility": "sterile", "in_count": false }, "t_s": 945.0, "count_event": null, "confidence": "medium", "note": "Scissors leave the field after cutting." }, { "layer": "L3", "object_id": "obj_hook", "object_class": "hook", "state": { "zone": "absent", "sterility": "sterile", "in_count": false }, "transition_from": { "zone": "in_field", "sterility": "sterile", "in_count": false }, "t_s": 1440.0, "count_event": null, "confidence": "medium", "note": "Hook leaves the field after gallbladder dissection completes." }, { "layer": "L3", "object_id": "obj_specimen_bag", "object_class": "specimen_bag", "state": { "zone": "in_field", "sterility": "sterile", "in_count": false }, "transition_from": null, "t_s": 1510.0, "count_event": null, "confidence": "medium", "note": "Specimen bag introduced for packaging; carries the gallbladder to retraction." }, { "layer": "L3", "object_id": "obj_irrigator", "object_class": "irrigator", "state": { "zone": "in_field", "sterility": "sterile", "in_count": false }, "transition_from": { "zone": "absent", "sterility": "sterile", "in_count": false }, "t_s": 1630.0, "count_event": null, "confidence": "medium", "note": "Irrigator present during cleaning and coagulation." } ], "L4": [] }, "rlds_excerpt": { "_note": "A 2-second derived RLDS slice (t=820.0-822.0 s) during the clipping-and-cutting phase, as the cystic artery is clipped. Derived view at 1 Hz decimation; the sidecar above remains the source of truth. context_events is empty because L4 is empty for this public-dataset source — the workflow-context layer is what Kindly adds. reward is 0.0: Kindly ships events, not reward shaping.", "episode_id": "kr-cholect50-cholecystectomy-demo-0001", "synthetic": false, "slice": { "t_start_s": 820.0, "t_end_s": 822.0, "decimation_hz": 1 }, "rlds_steps": [ { "t_s": 820.0, "is_first": false, "is_last": false, "is_terminal": false, "observation": { "phase": "clipping_and_cutting", "step": "clipping_and_cutting", "active_triplet": ["surgeon", "clip", "cystic_artery"], "object_states": { "obj_grasper": { "zone": "in_field", "sterility": "sterile", "in_count": false }, "obj_hook": { "zone": "in_field", "sterility": "sterile", "in_count": false }, "obj_clipper": { "zone": "in_field", "sterility": "sterile", "in_count": false } }, "context_events": [] }, "action": { "actor_role": "surgeon", "verb": "clip" }, "reward": 0.0, "discount": 1.0 }, { "t_s": 821.0, "is_first": false, "is_last": false, "is_terminal": false, "observation": { "phase": "clipping_and_cutting", "step": "clipping_and_cutting", "active_triplet": ["surgeon", "clip", "cystic_artery"], "object_states": { "obj_grasper": { "zone": "in_field", "sterility": "sterile", "in_count": false }, "obj_hook": { "zone": "in_field", "sterility": "sterile", "in_count": false }, "obj_clipper": { "zone": "in_field", "sterility": "sterile", "in_count": false } }, "context_events": [] }, "action": { "actor_role": "surgeon", "verb": "clip" }, "reward": 0.0, "discount": 1.0 } ] } }