trace-3d.github.io/index.html at main · trace-3d/trace-3d.github.io · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485

<!DOCTYPE html>
<html>
<head>
  <meta charset="utf-8">
  <meta name="description"
        content="Trace3D: Consistent Segmentation Lifting via Gaussian Instance Tracing">
  <meta name="keywords" content="DP-Recon">
  <meta name="viewport" content="width=device-width, initial-scale=1">
  <title>Trace3D</title>
  <!-- <link rel="icon" href="images/dprecon.png" type="image/png"> -->

  <script id="MathJax-script" async src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.7/MathJax.js?config=TeX-MML-AM_CHTML"></script>

    <link href="https://fonts.googleapis.com/css?family=Google+Sans|Noto+Sans|Castoro"
        rel="stylesheet">

  <link rel="stylesheet" href="./static/css/bulma.min.css">
  <link rel="stylesheet" href="./static/css/bulma-carousel.min.css">
  <link rel="stylesheet" href="./static/css/bulma-slider.min.css">
  <link rel="stylesheet" href="./static/css/fontawesome.all.min.css">
  <link rel="stylesheet" href="./static/css/index.css">
  <!-- <link rel="icon" href="./static/images/icon.png"> -->
  <link rel="stylesheet"
        href="https://cdn.jsdelivr.net/gh/jpswalsh/academicons@1/css/academicons.min.css">
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/swiper@11/swiper-bundle.min.css">


  <script src="./static/js/jquery.min.js"></script>
  <script defer src="./static/js/fontawesome.all.min.js"></script>
  <script src="./static/js/bulma-carousel.min.js"></script>
  <script src="./static/js/bulma-slider.min.js"></script>
  <script src="./static/js/index.js"></script>
  <script src="https://cdn.jsdelivr.net/npm/swiper@11/swiper-bundle.min.js"></script>

<style>
.grid-container {
  display: grid;
  grid-template-columns: auto auto auto auto auto auto;
  gap: 10px;
  padding: 0;
}

.grid-container > div {
  text-align: center;
  padding: 0;
  font-size: 30px;
}

.content h4 {
  font-size: 16px;
  padding: 0;
  margin: 0;
}

hr {
  background-color: lightgray;
}

</style>

</head>
<body>


<section class="hero">
  <style>
    .author-symbol {
      font-size: 1em;
      /* vertical-align: super; */
    }
    .author-block {
      margin: 0 0.2em;
    }
    .affiliation-note {
      font-size: 0.95em;
      margin-top: 0.5em;
    }
  </style>
  <div class="hero-body">
    <div class="container is-max-desktop">
      <div class="columns is-centered">
        <div class="column has-text-centered">
          <h1 class="title is-2 publication-title">Trace3D: Consistent Segmentation Lifting via Gaussian Instance Tracing</h1>
          <div class="is-size-4"><b>ICCV 2025</b></div>
          <!-- <br> -->
          <div class="is-size-5 publication-authors">
            <span class="author-block">
              <a >Hongyu Shen</a><sup>1,2<span class="author-symbol">*</span></sup>,</span>
            <span class="author-block">
              <a href="https://dali-jack.github.io/Junfeng-Ni/">Junfeng Ni</a><sup>2,3<span class="author-symbol">*</span></sup>,</span>
            <span class="author-block">
                <a href="https://yixchen.github.io/">Yixin Chen</a><sup>2<span class="author-symbol">✉</span></sup>,
            </span>
            <span class="author-block">
              <a >Weishuo Li</a><sup>2</sup>,
            </span>
            <span class="author-block">
              <a href="https://peimingtao.github.io/">Mingtao Pei</a><sup>1</sup>,
            </span>

            <span class="author-block">
              <a href="https://siyuanhuang.com/">Siyuan Huang</a><sup>2<span class="author-symbol">✉</span></sup>
            </span>

          </div>

          <div class="is-size-5 publication-authors">
            <span class="author-block"><sup>1</sup>Beijing Institute of Technology</span>
            <span class="author-block"><sup>2</sup>State Key Laboratory of General Artificial Intelligence, BIGAI </span>
            <span class="author-block"><sup>3</sup>Tsinghua University</span>
          </div>
          <div class="affiliation-note">
            <span class="author-symbol">*</span> Equal contribution &nbsp;&nbsp;
            <span class="author-symbol">✉</span> Corresponding author
          </div>
          <div class="column has-text-centered">
            <div class="publication-links">
              <span class="link-block">
                <a href="https://arxiv.org/pdf/2508.03227" target="_blank"
                   class="external-link button is-normal is-rounded is-dark">
                  <span class="icon">
                      <i class="fas fa-file-pdf"></i>
                  </span>
                  <span>Paper</span>
                </a>
              </span>
              <!-- Code Link. -->
               <span class="link-block">
                <a href="https://github.com/trace-3d/Trace3D" target="_blank"
		                class="external-link button is-normal is-rounded is-dark">
                  <span class="icon">
                      <i class="fab fa-github"></i>
                  </span>
                  <span >Code</span>
                  </a>
              </span>
              <!-- Video Link. -->
              <!-- <span class="link-block">
                <a href=""
                   class="external-link button is-normal is-rounded is-dark">
                  <span class="icon">
                      <i class="fab fa-youtube"></i>
                  </span>
                  <span>Video</span>
                </a>
              </span> -->
              <!-- Demo Video Link. -->
              <!-- <span class="link-block">
                <a href="https://youtu.be/QnJCoWiauro"
                   class="external-link button is-normal is-rounded is-dark">
                  <span class="icon">
                      <i class="fab fa-youtube"></i>
                  </span>
                  <span>Demo Video</span>
                </a>
              </span> -->
            </div>

          </div>
        </div>
    </div>
  </div>
</section>

<section class="hero teaser">
  <div class="container is-max-desktop">
    <h2 class="subtitle has-text-centered">
      <strong>Trace3D</strong> leverages the proposed <strong>Gaussian Instance Tracing</strong> to enhance multi-view consistency and reduce ambiguous Gaussians, resulting in high-quality 3D instance segmentation.
    </h2>
    <!-- <br> -->
    <!-- <p>DP-Recon reconstructs a high-quality interactive world from just 10 input views.
      Users can navigate, drag objects, edit geometry and texture via text, and apply photorealistic VFX.
      For the <strong>original, higher-quality video</strong>, please visit <a href="https://youtu.be/QnJCoWiauro">here</a>.
    </p> -->
    <br>
      <div class="columns is-centered">
        <video id="teaser" autoplay muted loop height="100%" controls>
          <source src="./static/videos/teaser.mp4"type="video/mp4">
        </video>
      </div>
      <br>
      <p>We visualize the feature map from PCA and
        hierarchical segmentations in each scene, facilitating object extraction and scene editing applications, e.g., Captain America wielding
        Thor's hammer.
      </p><br>
      <img src="./static/images/teaser.jpg" alt="">
      <br><br>
    </div>
  </div>
</section>

<section class="section">
  <div class="container is-max-desktop">
    <!-- Abstract. -->
    <div class="columns is-centered has-text-centered">
      <div class="columns is-centered has-text-centered">
        <h2 class="title is-3">Abstract</h2>
        </div>
        </div>
        <br>
        <div class="content has-text-justified">
          <p>
             We address the challenge of lifting 2D visual segmentation to 3D in Gaussian Splatting.
             Existing methods often suffer from inconsistent 2D masks across viewpoints and produce
             noisy segmentation boundaries as they neglect these semantic cues to refine the learned
             Gaussians. To overcome this, we introduce Gaussian Instance Tracing (GIT), which augments the standard Gaussian
             representation with an instance weight matrix across input views. Leveraging the inherent
             consistency of Gaussians in 3D, we use this matrix to identify and correct 2D segmentation
             inconsistencies. Furthermore, since each Gaussian ideally corresponds to a single object,
             we propose a GIT-guided adaptive density control mechanism to split and prune ambiguous
             Gaussians during training, resulting in sharper and more coherent 2D and 3D segmentation boundaries.
             Experimental results show that our method extracts clean 3D assets and consistently improves 3D segmentation
             in both online (e.g., self-prompting) and offline (e.g., contrastive lifting) settings, enabling applications
             such as hierarchical segmentation, object extraction, and scene editing.
          </p>
        </div>
      </div>
    </div>
  </div>
</section>

<section class="section">
  <div class="container is-max-desktop">
    <div class="columns is-centered">
      <!-- Visual Effects. -->
      <div class="column">
        <div class="content">
          <h4 class="title is-4">Mask inconsistency</h4>
          <p>
            The multi-view inconsistent 2D segmentation masks introduce significant ambiguity
            in both feature lifting and mask prediction.
          </p>
          <br>
          <img src="./static/images/mask-incon-mini.png" alt="">
        </div>
      </div>
      <!--/ Visual Effects. -->

      <!-- Matting. -->
      <div class="column">
        <h4 class="title is-4">Ambiguous gaussian</h4>
        <div class="columns is-centered">
          <div class="column content">
            <p>
              Gaussians nearby the edge often belong to both foreground and background at the same time,
              making precise 3D segmentation difficult.
            </p>
            <br>
            <img src="./static/images/ab.png" alt="">
          </div>

        </div>
      </div>
    </div>
</section>

<section class="section">
  <div class="container is-max-desktop">
    <div class="columns is-centered has-text-centered">
      <div class="columns is-centered has-text-centered">
        <h2 class="title is-3">Method</h2>
        </div>
        </div>
        <br>
        <div class="content has-text-justified">
          <p>
             We augments each 3D Gaussian with an instance weight matrix across views to capture its association with object instances.
             Leveraging the 3D consistency of Gaussians, GIT identifies and corrects 2D segmentation inconsistencies.
             Additionally, GIT-guided adaptive density control mechanism can split or prune ambiguous Gaussians, resuling clean and consistent instance boundaries in both 2D and 3D.
          </p>
          <!-- <img src="./images/method.png" alt=""> -->
          <video id="method" autoplay muted loop height="100%" controls>
            <source src="./static/videos/method.mp4" type="video/mp4">
          </video>
        </div>
      </div>
    </div>
  </div>
</section>

<section class="section" id="Result">
  <div class="container is-max-desktop">
    <div  class="columns is-centered has-text-centered">
      <div class="column is-four-fifths">
    <h2 class="title is-3">Results</h2>
    </div>
    </div>
    <br>

    <h3 class="title is-4">3D Object Extraction</h3>
        <div class="content has-text-justified">
          <p>
            Results on Replica demonstrate that our method not only accurately extracts the target object but also significantly reduces artifacts compared to the baselines.
         </p>
          <div class="container">
              <div id="results-carousel" class="carousel results-carousel">
                  <div class="item mesh_result-1">
                    <video poster="" id="steve" autoplay controls muted loop playsinline height="100%">
                      <source src="./static/videos/demo_video/replica/office_0_3_wo_side.mp4" type="video/mp4">
                    </video>
                  </div>
                  <div class="item mesh_result-2">
                    <video poster="" id="steve" autoplay controls muted loop playsinline height="100%">
                      <source src="./static/videos/demo_video/replica/office_0_9_wo_side.mp4" type="video/mp4">
                    </video>
                  </div>
                  <div class="item mesh_result-3">
                    <video poster="" id="steve" autoplay controls muted loop playsinline height="100%">
                      <source src="./static/videos/demo_video/replica/room_0_86_wo_side.mp4" type="video/mp4">
                    </video>
                  </div>
                  <div class="item mesh_result-4">
                    <video poster="" id="steve" autoplay controls muted loop playsinline height="100%">
                      <source src="./static/videos/demo_video/replica/room_2_38_wo_side.mp4" type="video/mp4">
                    </video>
                  </div>
              </div>
          </div>
        </div>

      <h3 class="title is-4">Novel View 2D Instance Segmentation</h3>
        <div class="content has-text-justified">
          <p>
            Our method also achieves higher-quality 2D instance segmentation results on both the Replica and NVOS datasets compared to other baselines.
            Our improvements in consistent instance maps and the reduction of ambiguous Gaussians yield more precise novel-view instance segmentation and feature maps, even for some tiny structures.
          </p>
          <h5 class="title is-5">Replica</h5>
          <img src="./static/images/NVS-Replica.png" alt="">
          <p>
            <strong>Novel view synthesis of 2D segmentation on Replica dataset.</strong> The results of each baseline provide the segmentation on the left and the feature map from PCA on the right.
            The <span style="color: yellow;">TP</span>,
            <span style="color: red;">FP</span>, and
            <span style="color: green;">FN</span> predictions are color-coded in the segmentation.
          </p>
          <h5 class="title is-5">NVOS</h5>
          <div class="container">
            <div id="results-carousel-3" class="carousel results-carousel">
              <div class="item trex">
                <video poster="" id="trex" autoplay controls muted loop playsinline height="100%">
                  <source src="./static/videos/demo_video/nvos/trex.mp4"
                          type="video/mp4">
                </video>
              </div>

              <div class="item horns-center">
                <video poster="" id="horns-center" autoplay controls muted loop playsinline height="100%">
                  <source src="./static/videos/demo_video/nvos/horns_center.mp4"
                          type="video/mp4">
                </video>
              </div>
              <div class="item flower">
                <video poster="" id="flower" autoplay controls muted loop playsinline height="100%">
                  <source src="./static/videos/demo_video/nvos/flower.mp4"
                          type="video/mp4">
                </video>
              </div>

              <div class="item orchids">
                <video poster="" id="orchids" autoplay controls muted loop playsinline height="100%">
                  <source src="./static/videos/demo_video/nvos/orchids.mp4"
                          type="video/mp4">
                </video>
              </div>

              <div class="item fortress">
                <video poster="" id="fortress" autoplay controls muted loop playsinline height="100%">
                  <source src="./static/videos/demo_video/nvos/fortress.mp4"
                          type="video/mp4">
                </video>
              </div>
              <div class="item fern">
                <video poster="" id="fern" autoplay controls muted loop playsinline height="100%">
                  <source src="./static/videos/demo_video/nvos/fern.mp4"
                          type="video/mp4">
                </video>
              </div>
              <div class="item leaves">
                <video poster="" id="leaves" autoplay controls muted loop playsinline height="100%">
                  <source src="./static/videos/demo_video/nvos/leaves.mp4"
                          type="video/mp4">
                </video>
              </div>
              <div class="item horns-left">
                <video poster="" id="horns-left" autoplay controls muted loop playsinline height="100%">
                  <source src="./static/videos/demo_video/nvos/horns_left.mp4"
                          type="video/mp4">
                </video>
              </div>
            </div>
            <p>
              <strong>Novel view synthesis of 2D segmentation on NVOS dataset.</strong>
              The <span style="color: yellow;">TP</span>,
              <span style="color: red;">FP</span>, and
              <span style="color: green;">FN</span> predictions are color-coded in the segmentation.
              Our results with GIT produce fewer false positive predictions that are hard to observe from the input view.
            </p>

        </div>
        </div>

        <h3 class="title is-4">Part-level Segmentation</h3>
        <div class="content has-text-justified">
          <p>
           The videos below show how our method enables clean 3D segmentation and flexible object extraction, supporting various downstream applications.
         </p>

          <div class="container">
            <div id="results-carousel" class="carousel results-carousel">
              <div class="item mesh_result-1">
                <video poster="" id="steve" autoplay controls muted loop playsinline height="100%">
                  <source src="./static/videos/logo.mp4" type="video/mp4">
                </video>
              </div>
              <div class="item mesh_result-4">
                <video poster="" id="steve" autoplay controls muted loop playsinline height="100%">
                  <source src="./static/videos/camera.mp4" type="video/mp4">
                </video>
              </div>

            </div>
        </div>
        </div>


<!-- <br> -->

<section class="section" id="BibTeX">
  <div class="container is-max-desktop content">
    <h2 class="title">Related Work</h2>
      <a href="http://arxiv.org/abs/2306.04633">Contrastive Lift: 3D Object Instance Segmentation by Slow-Fast Contrastive Fusion</a>
      <br>
      <a href="http://arxiv.org/abs/2304.12308">Segment Anything in 3D with Radiance Fields</a>
      <br>
      <a href="https://arxiv.org/abs/2311.11666">OmniSeg3D: Omniversal 3D Segmentation via  Hierarchical Contrastive Learning</a>
      <br>
      <a href="http://arxiv.org/abs/2403.18118">EgoLifter: Open-world 3D Segmentation for Egocentric Perception</a>
      <br>
      <a href="http://arxiv.org/abs/2409.08270">FlashSplat: 2D to 3D Gaussian Splatting Segmentation Solved Optimally</a>
      <br>
      <a href="http://arxiv.org/abs/2411.07555">GaussianCut: Interactive segmentation via graph cut for 3D Gaussian Splatting</a>
  </div>
</section>

<section class="section" id="BibTeX">
    <div class="container is-max-desktop content">
      <h2 class="title">BibTeX</h2>
      <pre><code>@inproceedings{shen2025trace3d,
  title={Trace3D: Consistent Segmentation Lifting via Gaussian Instance Tracing},
  author={Shen, Hongyu and Ni, Junfeng and Chen, Yixin and Li, Weishuo and Pei, Mingtao and Huang, Siyuan},
  booktitle=ICCV,
  year={2025}
}</code></pre>
    </div>
  </section>


<footer class="footer">
    <div class="columns is-centered">
      <div class="column is-8">
        <div class="content">
          <p>
            This webpage originated from <a href="https://nerfies.github.io/">Nerfies</a>.
            This website is licensed under a <a rel="license"
                                                href="http://creativecommons.org/licenses/by-sa/4.0/">Creative
            Commons Attribution-ShareAlike 4.0 International License</a>.
          </p>
        </div>
      </div>
    </div>
</footer>

<script>
  bulmaCarousel.attach('#results-carousel-3', {
    slidesToShow: 3,
    slidesToScroll: 1,
    navigation: true,
    pagination: true,
    loop: true,
    breakpoints: []
  });
</script>

</body>
</html>