Upload 186 files

.gitattributes

@@ -33,3 +33,67 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/T.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_building_building.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_building_castle.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_building_colorful_cottage.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_building_maya_pyramid.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_building_mushroom.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_building_space_station.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_creature_dragon.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_creature_elephant.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_creature_furry.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_creature_quadruped.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_creature_robot_crab.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_creature_robot_dinosour.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_creature_rock_monster.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_humanoid_block_robot.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_humanoid_dragonborn.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_humanoid_dwarf.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_humanoid_goblin.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_humanoid_mech.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_misc_crate.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_misc_fireplace.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_misc_gate.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_misc_lantern.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_misc_magicbook.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_misc_mailbox.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_misc_monster_chest.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_misc_paper_machine.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_misc_phonograph.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_misc_portal2.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_misc_storage_chest.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_misc_telephone.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_misc_television.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_misc_workbench.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_vehicle_biplane.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_vehicle_bulldozer.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_vehicle_cart.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_vehicle_excavator.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_vehicle_helicopter.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_vehicle_locomotive.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/typical_vehicle_pirate_ship.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_image/weatherworn_misc_paper_machine3.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_multi_image/character_1.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_multi_image/character_2.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_multi_image/character_3.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_multi_image/mushroom_1.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_multi_image/mushroom_2.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_multi_image/mushroom_3.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_multi_image/orangeguy_1.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_multi_image/orangeguy_2.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_multi_image/orangeguy_3.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_multi_image/popmart_1.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_multi_image/popmart_2.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_multi_image/popmart_3.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_multi_image/rabbit_1.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_multi_image/rabbit_2.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_multi_image/rabbit_3.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_multi_image/tiger_1.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_multi_image/tiger_2.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_multi_image/tiger_3.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_multi_image/yoimiya_1.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_multi_image/yoimiya_2.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/example_multi_image/yoimiya_3.png filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/logo.webp filter=lfs diff=lfs merge=lfs -text
+TRELLIS/assets/teaser.png filter=lfs diff=lfs merge=lfs -text

TRELLIS/.gitignore

@@ -0,0 +1,398 @@
+## Ignore Visual Studio temporary files, build results, and
+## files generated by popular Visual Studio add-ons.
+##
+## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore
+
+# User-specific files
+*.rsuser
+*.suo
+*.user
+*.userosscache
+*.sln.docstates
+
+# User-specific files (MonoDevelop/Xamarin Studio)
+*.userprefs
+
+# Mono auto generated files
+mono_crash.*
+
+# Build results
+[Dd]ebug/
+[Dd]ebugPublic/
+[Rr]elease/
+[Rr]eleases/
+x64/
+x86/
+[Ww][Ii][Nn]32/
+[Aa][Rr][Mm]/
+[Aa][Rr][Mm]64/
+bld/
+[Bb]in/
+[Oo]bj/
+[Ll]og/
+[Ll]ogs/
+
+# Visual Studio 2015/2017 cache/options directory
+.vs/
+# Uncomment if you have tasks that create the project's static files in wwwroot
+#wwwroot/
+
+# Visual Studio 2017 auto generated files
+Generated\ Files/
+
+# MSTest test Results
+[Tt]est[Rr]esult*/
+[Bb]uild[Ll]og.*
+
+# NUnit
+*.VisualState.xml
+TestResult.xml
+nunit-*.xml
+
+# Build Results of an ATL Project
+[Dd]ebugPS/
+[Rr]eleasePS/
+dlldata.c
+
+# Benchmark Results
+BenchmarkDotNet.Artifacts/
+
+# .NET Core
+project.lock.json
+project.fragment.lock.json
+artifacts/
+
+# ASP.NET Scaffolding
+ScaffoldingReadMe.txt
+
+# StyleCop
+StyleCopReport.xml
+
+# Files built by Visual Studio
+*_i.c
+*_p.c
+*_h.h
+*.ilk
+*.meta
+*.obj
+*.iobj
+*.pch
+*.pdb
+*.ipdb
+*.pgc
+*.pgd
+*.rsp
+*.sbr
+*.tlb
+*.tli
+*.tlh
+*.tmp
+*.tmp_proj
+*_wpftmp.csproj
+*.log
+*.tlog
+*.vspscc
+*.vssscc
+.builds
+*.pidb
+*.svclog
+*.scc
+
+# Chutzpah Test files
+_Chutzpah*
+
+# Visual C++ cache files
+ipch/
+*.aps
+*.ncb
+*.opendb
+*.opensdf
+*.sdf
+*.cachefile
+*.VC.db
+*.VC.VC.opendb
+
+# Visual Studio profiler
+*.psess
+*.vsp
+*.vspx
+*.sap
+
+# Visual Studio Trace Files
+*.e2e
+
+# TFS 2012 Local Workspace
+$tf/
+
+# Guidance Automation Toolkit
+*.gpState
+
+# ReSharper is a .NET coding add-in
+_ReSharper*/
+*.[Rr]e[Ss]harper
+*.DotSettings.user
+
+# TeamCity is a build add-in
+_TeamCity*
+
+# DotCover is a Code Coverage Tool
+*.dotCover
+
+# AxoCover is a Code Coverage Tool
+.axoCover/*
+!.axoCover/settings.json
+
+# Coverlet is a free, cross platform Code Coverage Tool
+coverage*.json
+coverage*.xml
+coverage*.info
+
+# Visual Studio code coverage results
+*.coverage
+*.coveragexml
+
+# NCrunch
+_NCrunch_*
+.*crunch*.local.xml
+nCrunchTemp_*
+
+# MightyMoose
+*.mm.*
+AutoTest.Net/
+
+# Web workbench (sass)
+.sass-cache/
+
+# Installshield output folder
+[Ee]xpress/
+
+# DocProject is a documentation generator add-in
+DocProject/buildhelp/
+DocProject/Help/*.HxT
+DocProject/Help/*.HxC
+DocProject/Help/*.hhc
+DocProject/Help/*.hhk
+DocProject/Help/*.hhp
+DocProject/Help/Html2
+DocProject/Help/html
+
+# Click-Once directory
+publish/
+
+# Publish Web Output
+*.[Pp]ublish.xml
+*.azurePubxml
+# Note: Comment the next line if you want to checkin your web deploy settings,
+# but database connection strings (with potential passwords) will be unencrypted
+*.pubxml
+*.publishproj
+
+# Microsoft Azure Web App publish settings. Comment the next line if you want to
+# checkin your Azure Web App publish settings, but sensitive information contained
+# in these scripts will be unencrypted
+PublishScripts/
+
+# NuGet Packages
+*.nupkg
+# NuGet Symbol Packages
+*.snupkg
+# The packages folder can be ignored because of Package Restore
+**/[Pp]ackages/*
+# except build/, which is used as an MSBuild target.
+!**/[Pp]ackages/build/
+# Uncomment if necessary however generally it will be regenerated when needed
+#!**/[Pp]ackages/repositories.config
+# NuGet v3's project.json files produces more ignorable files
+*.nuget.props
+*.nuget.targets
+
+# Microsoft Azure Build Output
+csx/
+*.build.csdef
+
+# Microsoft Azure Emulator
+ecf/
+rcf/
+
+# Windows Store app package directories and files
+AppPackages/
+BundleArtifacts/
+Package.StoreAssociation.xml
+_pkginfo.txt
+*.appx
+*.appxbundle
+*.appxupload
+
+# Visual Studio cache files
+# files ending in .cache can be ignored
+*.[Cc]ache
+# but keep track of directories ending in .cache
+!?*.[Cc]ache/
+
+# Others
+ClientBin/
+~$*
+*~
+*.dbmdl
+*.dbproj.schemaview
+*.jfm
+*.pfx
+*.publishsettings
+orleans.codegen.cs
+
+# Including strong name files can present a security risk
+# (https://github.com/github/gitignore/pull/2483#issue-259490424)
+#*.snk
+
+# Since there are multiple workflows, uncomment next line to ignore bower_components
+# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
+#bower_components/
+
+# RIA/Silverlight projects
+Generated_Code/
+
+# Backup & report files from converting an old project file
+# to a newer Visual Studio version. Backup files are not needed,
+# because we have git ;-)
+_UpgradeReport_Files/
+Backup*/
+UpgradeLog*.XML
+UpgradeLog*.htm
+ServiceFabricBackup/
+*.rptproj.bak
+
+# SQL Server files
+*.mdf
+*.ldf
+*.ndf
+
+# Business Intelligence projects
+*.rdl.data
+*.bim.layout
+*.bim_*.settings
+*.rptproj.rsuser
+*- [Bb]ackup.rdl
+*- [Bb]ackup ([0-9]).rdl
+*- [Bb]ackup ([0-9][0-9]).rdl
+
+# Microsoft Fakes
+FakesAssemblies/
+
+# GhostDoc plugin setting file
+*.GhostDoc.xml
+
+# Node.js Tools for Visual Studio
+.ntvs_analysis.dat
+node_modules/
+
+# Visual Studio 6 build log
+*.plg
+
+# Visual Studio 6 workspace options file
+*.opt
+
+# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
+*.vbw
+
+# Visual Studio 6 auto-generated project file (contains which files were open etc.)
+*.vbp
+
+# Visual Studio 6 workspace and project file (working project files containing files to include in project)
+*.dsw
+*.dsp
+
+# Visual Studio 6 technical files
+*.ncb
+*.aps
+
+# Visual Studio LightSwitch build output
+**/*.HTMLClient/GeneratedArtifacts
+**/*.DesktopClient/GeneratedArtifacts
+**/*.DesktopClient/ModelManifest.xml
+**/*.Server/GeneratedArtifacts
+**/*.Server/ModelManifest.xml
+_Pvt_Extensions
+
+# Paket dependency manager
+.paket/paket.exe
+paket-files/
+
+# FAKE - F# Make
+.fake/
+
+# CodeRush personal settings
+.cr/personal
+
+# Python Tools for Visual Studio (PTVS)
+__pycache__/
+*.pyc
+
+# Cake - Uncomment if you are using it
+# tools/**
+# !tools/packages.config
+
+# Tabs Studio
+*.tss
+
+# Telerik's JustMock configuration file
+*.jmconfig
+
+# BizTalk build output
+*.btp.cs
+*.btm.cs
+*.odx.cs
+*.xsd.cs
+
+# OpenCover UI analysis results
+OpenCover/
+
+# Azure Stream Analytics local run output
+ASALocalRun/
+
+# MSBuild Binary and Structured Log
+*.binlog
+
+# NVidia Nsight GPU debugger configuration file
+*.nvuser
+
+# MFractors (Xamarin productivity tool) working folder
+.mfractor/
+
+# Local History for Visual Studio
+.localhistory/
+
+# Visual Studio History (VSHistory) files
+.vshistory/
+
+# BeatPulse healthcheck temp database
+healthchecksdb
+
+# Backup folder for Package Reference Convert tool in Visual Studio 2017
+MigrationBackup/
+
+# Ionide (cross platform F# VS Code tools) working folder
+.ionide/
+
+# Fody - auto-generated XML schema
+FodyWeavers.xsd
+
+# VS Code files for those working on multiple tools
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+*.code-workspace
+
+# Local History for Visual Studio Code
+.history/
+
+# Windows Installer files from build outputs
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# JetBrains Rider
+*.sln.iml

TRELLIS/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "trellis/representations/mesh/flexicubes"]
+	path = trellis/representations/mesh/flexicubes
+	url = https://github.com/MaxtirError/FlexiCubes.git

TRELLIS/CODE_OF_CONDUCT.md

@@ -0,0 +1,9 @@
+# Microsoft Open Source Code of Conduct
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
+
+Resources:
+
+- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/)
+- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/)
+- Contact [[email protected]](mailto:[email protected]) with questions or concerns

TRELLIS/DATASET.md

@@ -0,0 +1,231 @@
+# TRELLIS-500K
+
+TRELLIS-500K is a dataset of 500K 3D assets curated from [Objaverse(XL)](https://objaverse.allenai.org/), [ABO](https://amazon-berkeley-objects.s3.amazonaws.com/index.html), [3D-FUTURE](https://tianchi.aliyun.com/specials/promotion/alibaba-3d-future), [HSSD](https://huggingface.co/datasets/hssd/hssd-models), and [Toys4k](https://github.com/rehg-lab/lowshot-shapebias/tree/main/toys4k), filtered based on aesthetic scores.
+This dataset serves for 3D generation tasks.
+
+The dataset is provided as csv files containing the 3D assets' metadata.
+
+## Dataset Statistics
+
+The following table summarizes the dataset's filtering and composition:
+
+***NOTE: Some of the 3D assets lack text captions. Please filter out such assets if captions are required.***
+| Source | Aesthetic Score Threshold | Filtered Size | With Captions |
+|:-:|:-:|:-:|:-:|
+| ObjaverseXL (sketchfab) | 5.5 | 168307 | 167638 |
+| ObjaverseXL (github) | 5.5 | 311843 | 306790 |
+| ABO | 4.5 | 4485 | 4390 |
+| 3D-FUTURE | 4.5 | 9472 | 9291 |
+| HSSD | 4.5 | 6670 | 6661 |
+| All (training set) | - | 500777 | 494770 |
+| Toys4k (evaluation set) | 4.5 | 3229 | 3180 |
+
+## Dataset Location
+
+The dataset is hosted on Hugging Face Datasets. You can preview the dataset at
+
+[https://huggingface.co/datasets/JeffreyXiang/TRELLIS-500K](https://huggingface.co/datasets/JeffreyXiang/TRELLIS-500K)
+
+There is no need to download the csv files manually. We provide toolkits to load and prepare the dataset.
+
+## Dataset Toolkits
+
+We provide [toolkits](dataset_toolkits) for data preparation.
+
+### Step 1: Install Dependencies
+
+```
+. ./dataset_toolkits/setup.sh
+```
+
+### Step 2: Load Metadata
+
+First, we need to load the metadata of the dataset.
+
+```
+python dataset_toolkits/build_metadata.py <SUBSET> --output_dir <OUTPUT_DIR> [--source <SOURCE>]
+```
+
+- `SUBSET`: The subset of the dataset to load. Options are `ObjaverseXL`, `ABO`, `3D-FUTURE`, `HSSD`, and `Toys4k`.
+- `OUTPUT_DIR`: The directory to save the data.
+- `SOURCE`: Required if `SUBSET` is `ObjaverseXL`. Options are `sketchfab` and `github`.
+
+For example, to load the metadata of the ObjaverseXL (sketchfab) subset and save it to `datasets/ObjaverseXL_sketchfab`, we can run:
+
+```
+python dataset_toolkits/build_metadata.py ObjaverseXL --source sketchfab --output_dir datasets/ObjaverseXL_sketchfab
+```
+
+### Step 3: Download Data
+
+Next, we need to download the 3D assets.
+
+```
+python dataset_toolkits/download.py <SUBSET> --output_dir <OUTPUT_DIR> [--rank <RANK> --world_size <WORLD_SIZE>]
+```
+
+- `SUBSET`: The subset of the dataset to download. Options are `ObjaverseXL`, `ABO`, `3D-FUTURE`, `HSSD`, and `Toys4k`.
+- `OUTPUT_DIR`: The directory to save the data.
+
+You can also specify the `RANK` and `WORLD_SIZE` of the current process if you are using multiple nodes for data preparation.
+
+For example, to download the ObjaverseXL (sketchfab) subset and save it to `datasets/ObjaverseXL_sketchfab`, we can run: 
+
+***NOTE: The example command below sets a large `WORLD_SIZE` for demonstration purposes. Only a small portion of the dataset will be downloaded.***
+
+```
+python dataset_toolkits/download.py ObjaverseXL --output_dir datasets/ObjaverseXL_sketchfab --world_size 160000
+```
+
+Some datasets may require interactive login to Hugging Face or manual downloading. Please follow the instructions given by the toolkits.
+
+After downloading, update the metadata file with:
+
+```
+python dataset_toolkits/build_metadata.py ObjaverseXL --output_dir datasets/ObjaverseXL_sketchfab
+```
+
+### Step 4: Render Multiview Images
+
+Multiview images can be rendered with:
+
+```
+python dataset_toolkits/render.py <SUBSET> --output_dir <OUTPUT_DIR> [--num_views <NUM_VIEWS>] [--rank <RANK> --world_size <WORLD_SIZE>]
+```
+
+- `SUBSET`: The subset of the dataset to render. Options are `ObjaverseXL`, `ABO`, `3D-FUTURE`, `HSSD`, and `Toys4k`.
+- `OUTPUT_DIR`: The directory to save the data.
+- `NUM_VIEWS`: The number of views to render. Default is 150.
+- `RANK` and `WORLD_SIZE`: Multi-node configuration.
+
+For example, to render the ObjaverseXL (sketchfab) subset and save it to `datasets/ObjaverseXL_sketchfab`, we can run:
+
+```
+python dataset_toolkits/render.py ObjaverseXL --output_dir datasets/ObjaverseXL_sketchfab
+```
+
+Don't forget to update the metadata file with:
+
+```
+python dataset_toolkits/build_metadata.py ObjaverseXL --output_dir datasets/ObjaverseXL_sketchfab
+```
+
+### Step 5: Voxelize 3D Models
+
+We can voxelize the 3D models with:
+
+```
+python dataset_toolkits/voxelize.py <SUBSET> --output_dir <OUTPUT_DIR> [--rank <RANK> --world_size <WORLD_SIZE>]
+```
+
+- `SUBSET`: The subset of the dataset to voxelize. Options are `ObjaverseXL`, `ABO`, `3D-FUTURE`, `HSSD`, and `Toys4k`.
+- `OUTPUT_DIR`: The directory to save the data.
+- `RANK` and `WORLD_SIZE`: Multi-node configuration.
+
+For example, to voxelize the ObjaverseXL (sketchfab) subset and save it to `datasets/ObjaverseXL_sketchfab`, we can run:
+```
+python dataset_toolkits/voxelize.py ObjaverseXL --output_dir datasets/ObjaverseXL_sketchfab
+```
+
+Then update the metadata file with:
+
+```
+python dataset_toolkits/build_metadata.py ObjaverseXL --output_dir datasets/ObjaverseXL_sketchfab
+```
+
+### Step 6: Extract DINO Features
+
+To prepare the training data for SLat VAE, we need to extract DINO features from multiview images and aggregate them into sparse voxel grids.
+
+```
+python dataset_toolkits/extract_features.py --output_dir <OUTPUT_DIR> [--rank <RANK> --world_size <WORLD_SIZE>]
+```
+
+- `OUTPUT_DIR`: The directory to save the data.
+- `RANK` and `WORLD_SIZE`: Multi-node configuration.
+
+
+For example, to extract DINO features from the ObjaverseXL (sketchfab) subset and save it to `datasets/ObjaverseXL_sketchfab`, we can run:
+
+```
+python dataset_toolkits/extract_feature.py --output_dir datasets/ObjaverseXL_sketchfab
+```
+
+Then update the metadata file with:
+
+```
+python dataset_toolkits/build_metadata.py ObjaverseXL --output_dir datasets/ObjaverseXL_sketchfab
+```
+
+### Step 7: Encode Sparse Structures
+
+Encoding the sparse structures into latents to train the first stage generator:
+
+```
+python dataset_toolkits/encode_ss_latent.py --output_dir <OUTPUT_DIR> [--rank <RANK> --world_size <WORLD_SIZE>]
+```
+
+- `OUTPUT_DIR`: The directory to save the data.
+- `RANK` and `WORLD_SIZE`: Multi-node configuration.
+
+For example, to encode the sparse structures into latents for the ObjaverseXL (sketchfab) subset and save it to `datasets/ObjaverseXL_sketchfab`, we can run:
+
+```
+python dataset_toolkits/encode_ss_latent.py --output_dir datasets/ObjaverseXL_sketchfab
+```
+
+Then update the metadata file with:
+
+```
+python dataset_toolkits/build_metadata.py ObjaverseXL --output_dir datasets/ObjaverseXL_sketchfab
+```
+
+### Step 8: Encode SLat
+
+Encoding SLat for second stage generator training:
+
+```
+python dataset_toolkits/encode_latent.py --output_dir <OUTPUT_DIR> [--rank <RANK> --world_size <WORLD_SIZE>]
+```
+
+- `OUTPUT_DIR`: The directory to save the data.
+- `RANK` and `WORLD_SIZE`: Multi-node configuration.
+
+For example, to encode SLat for the ObjaverseXL (sketchfab) subset and save it to `datasets/ObjaverseXL_sketchfab`, we can run:
+
+```
+python dataset_toolkits/encode_latent.py --output_dir datasets/ObjaverseXL_sketchfab
+```
+
+Then update the metadata file with:
+
+```
+python dataset_toolkits/build_metadata.py ObjaverseXL --output_dir datasets/ObjaverseXL_sketchfab
+```
+
+### Step 9: Render Image Conditions
+
+To train the image conditioned generator, we need to render image conditions with augmented views.
+
+```
+python dataset_toolkits/render_cond.py <SUBSET> --output_dir <OUTPUT_DIR> [--num_views <NUM_VIEWS>] [--rank <RANK> --world_size <WORLD_SIZE>]
+```
+
+- `SUBSET`: The subset of the dataset to render. Options are `ObjaverseXL`, `ABO`, `3D-FUTURE`, `HSSD`, and `Toys4k`.
+- `OUTPUT_DIR`: The directory to save the data.
+- `NUM_VIEWS`: The number of views to render. Default is 24.
+- `RANK` and `WORLD_SIZE`: Multi-node configuration.
+
+For example, to render image conditions for the ObjaverseXL (sketchfab) subset and save it to `datasets/ObjaverseXL_sketchfab`, we can run:
+
+```
+python dataset_toolkits/render_cond.py ObjaverseXL --output_dir datasets/ObjaverseXL_sketchfab
+```
+
+Then update the metadata file with:
+
+```
+python dataset_toolkits/build_metadata.py ObjaverseXL --output_dir datasets/ObjaverseXL_sketchfab
+```
+
+

TRELLIS/LICENSE

@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE

TRELLIS/README.md

@@ -0,0 +1,230 @@
+<img src="assets/logo.webp" width="100%" align="center">
+<h1 align="center">Structured 3D Latents<br>for Scalable and Versatile 3D Generation</h1>
+<p align="center"><a href="https://arxiv.org/abs/2412.01506"><img src='https://img.shields.io/badge/arXiv-Paper-red?logo=arxiv&logoColor=white' alt='arXiv'></a>
+<a href='https://trellis3d.github.io'><img src='https://img.shields.io/badge/Project_Page-Website-green?logo=googlechrome&logoColor=white' alt='Project Page'></a>
+<a href='https://huggingface.co/spaces/JeffreyXiang/TRELLIS'><img src='https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Live_Demo-blue'></a>
+</p>
+<p align="center"><img src="assets/teaser.png" width="100%"></p>
+
+<span style="font-size: 16px; font-weight: 600;">T</span><span style="font-size: 12px; font-weight: 700;">RELLIS</span> is a large 3D asset generation model. It takes in text or image prompts and generates high-quality 3D assets in various formats, such as Radiance Fields, 3D Gaussians, and meshes. The cornerstone of <span style="font-size: 16px; font-weight: 600;">T</span><span style="font-size: 12px; font-weight: 700;">RELLIS</span> is a unified Structured LATent (<span style="font-size: 16px; font-weight: 600;">SL</span><span style="font-size: 12px; font-weight: 700;">AT</span>) representation that allows decoding to different output formats and Rectified Flow Transformers tailored for <span style="font-size: 16px; font-weight: 600;">SL</span><span style="font-size: 12px; font-weight: 700;">AT</span> as the powerful backbones. We provide large-scale pre-trained models with up to 2 billion parameters on a large 3D asset dataset of 500K diverse objects. <span style="font-size: 16px; font-weight: 600;">T</span><span style="font-size: 12px; font-weight: 700;">RELLIS</span> significantly surpasses existing methods, including recent ones at similar scales, and showcases flexible output format selection and local 3D editing capabilities which were not offered by previous models.
+
+***Check out our [Project Page](https://trellis3d.github.io) for more videos and interactive demos!***
+
+<!-- Features -->
+## 🌟 Features
+- **High Quality**: It produces diverse 3D assets at high quality with intricate shape and texture details.
+- **Versatility**: It takes text or image prompts and can generate various final 3D representations including but not limited to *Radiance Fields*, *3D Gaussians*, and *meshes*, accommodating diverse downstream requirements.
+- **Flexible Editing**: It allows for easy editings of generated 3D assets, such as generating variants of the same object or local editing of the 3D asset.
+
+<!-- Updates -->
+## ⏩ Updates
+
+**12/26/2024**
+- Release [**TRELLIS-500K**](https://github.com/microsoft/TRELLIS#-dataset) dataset and toolkits for data preparation.
+
+**12/18/2024**
+- Implementation of multi-image conditioning for TRELLIS-image model. ([#7](https://github.com/microsoft/TRELLIS/issues/7)). This is based on tuning-free algorithm without training a specialized model, so it may not give the best results for all input images.
+- Add Gaussian export in `app.py` and `example.py`. ([#40](https://github.com/microsoft/TRELLIS/issues/40))
+
+<!-- TODO List -->
+## 🚧 TODO List
+- [x] Release inference code and TRELLIS-image-large model
+- [x] Release dataset and dataset toolkits
+- [ ] Release TRELLIS-text model series
+- [ ] Release training code
+
+<!-- Installation -->
+## 📦 Installation
+
+### Prerequisites
+- **System**: The code is currently tested only on **Linux**.  For windows setup, you may refer to [#3](https://github.com/microsoft/TRELLIS/issues/3) (not fully tested).
+- **Hardware**: An NVIDIA GPU with at least 16GB of memory is necessary. The code has been verified on NVIDIA A100 and A6000 GPUs.  
+- **Software**:   
+  - The [CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit-archive) is needed to compile certain submodules. The code has been tested with CUDA versions 11.8 and 12.2.  
+  - [Conda](https://docs.anaconda.com/miniconda/install/#quick-command-line-install) is recommended for managing dependencies.  
+  - Python version 3.8 or higher is required. 
+
+### Installation Steps
+1. Clone the repo:
+    ```sh
+    git clone --recurse-submodules https://github.com/microsoft/TRELLIS.git
+    cd TRELLIS
+    ```
+
+2. Install the dependencies:
+    
+    **Before running the following command there are somethings to note:**
+    - By adding `--new-env`, a new conda environment named `trellis` will be created. If you want to use an existing conda environment, please remove this flag.
+    - By default the `trellis` environment will use pytorch 2.4.0 with CUDA 11.8. If you want to use a different version of CUDA (e.g., if you have CUDA Toolkit 12.2 installed and do not want to install another 11.8 version for submodule compilation), you can remove the `--new-env` flag and manually install the required dependencies. Refer to [PyTorch](https://pytorch.org/get-started/previous-versions/) for the installation command.
+    - If you have multiple CUDA Toolkit versions installed, `PATH` should be set to the correct version before running the command. For example, if you have CUDA Toolkit 11.8 and 12.2 installed, you should run `export PATH=/usr/local/cuda-11.8/bin:$PATH` before running the command.
+    - By default, the code uses the `flash-attn` backend for attention. For GPUs do not support `flash-attn` (e.g., NVIDIA V100), you can remove the `--flash-attn` flag to install `xformers` only and set the `ATTN_BACKEND` environment variable to `xformers` before running the code. See the [Minimal Example](#minimal-example) for more details.
+    - The installation may take a while due to the large number of dependencies. Please be patient. If you encounter any issues, you can try to install the dependencies one by one, specifying one flag at a time.
+    - If you encounter any issues during the installation, feel free to open an issue or contact us.
+    
+    Create a new conda environment named `trellis` and install the dependencies:
+    ```sh
+    . ./setup.sh --new-env --basic --xformers --flash-attn --diffoctreerast --spconv --mipgaussian --kaolin --nvdiffrast
+    ```
+    The detailed usage of `setup.sh` can be found by running `. ./setup.sh --help`.
+    ```sh
+    Usage: setup.sh [OPTIONS]
+    Options:
+        -h, --help              Display this help message
+        --new-env               Create a new conda environment
+        --basic                 Install basic dependencies
+        --xformers              Install xformers
+        --flash-attn            Install flash-attn
+        --diffoctreerast        Install diffoctreerast
+        --vox2seq               Install vox2seq
+        --spconv                Install spconv
+        --mipgaussian           Install mip-splatting
+        --kaolin                Install kaolin
+        --nvdiffrast            Install nvdiffrast
+        --demo                  Install all dependencies for demo
+    ```
+
+<!-- Pretrained Models -->
+## 🤖 Pretrained Models
+
+We provide the following pretrained models:
+
+| Model | Description | #Params | Download |
+| --- | --- | --- | --- |
+| TRELLIS-image-large | Large image-to-3D model | 1.2B | [Download](https://huggingface.co/JeffreyXiang/TRELLIS-image-large) |
+| TRELLIS-text-base | Base text-to-3D model | 342M | Coming Soon |
+| TRELLIS-text-large | Large text-to-3D model | 1.1B | Coming Soon |
+| TRELLIS-text-xlarge | Extra-large text-to-3D model | 2.0B | Coming Soon |
+
+The models are hosted on Hugging Face. You can directly load the models with their repository names in the code:
+```python
+TrellisImageTo3DPipeline.from_pretrained("JeffreyXiang/TRELLIS-image-large")
+```
+
+If you prefer loading the model from local, you can download the model files from the links above and load the model with the folder path (folder structure should be maintained):
+```python
+TrellisImageTo3DPipeline.from_pretrained("/path/to/TRELLIS-image-large")
+```
+
+<!-- Usage -->
+## 💡 Usage
+
+### Minimal Example
+
+Here is an [example](example.py) of how to use the pretrained models for 3D asset generation.
+
+```python
+import os
+# os.environ['ATTN_BACKEND'] = 'xformers'   # Can be 'flash-attn' or 'xformers', default is 'flash-attn'
+os.environ['SPCONV_ALGO'] = 'native'        # Can be 'native' or 'auto', default is 'auto'.
+                                            # 'auto' is faster but will do benchmarking at the beginning.
+                                            # Recommended to set to 'native' if run only once.
+
+import imageio
+from PIL import Image
+from trellis.pipelines import TrellisImageTo3DPipeline
+from trellis.utils import render_utils, postprocessing_utils
+
+# Load a pipeline from a model folder or a Hugging Face model hub.
+pipeline = TrellisImageTo3DPipeline.from_pretrained("JeffreyXiang/TRELLIS-image-large")
+pipeline.cuda()
+
+# Load an image
+image = Image.open("assets/example_image/T.png")
+
+# Run the pipeline
+outputs = pipeline.run(
+    image,
+    seed=1,
+    # Optional parameters
+    # sparse_structure_sampler_params={
+    #     "steps": 12,
+    #     "cfg_strength": 7.5,
+    # },
+    # slat_sampler_params={
+    #     "steps": 12,
+    #     "cfg_strength": 3,
+    # },
+)
+# outputs is a dictionary containing generated 3D assets in different formats:
+# - outputs['gaussian']: a list of 3D Gaussians
+# - outputs['radiance_field']: a list of radiance fields
+# - outputs['mesh']: a list of meshes
+
+# Render the outputs
+video = render_utils.render_video(outputs['gaussian'][0])['color']
+imageio.mimsave("sample_gs.mp4", video, fps=30)
+video = render_utils.render_video(outputs['radiance_field'][0])['color']
+imageio.mimsave("sample_rf.mp4", video, fps=30)
+video = render_utils.render_video(outputs['mesh'][0])['normal']
+imageio.mimsave("sample_mesh.mp4", video, fps=30)
+
+# GLB files can be extracted from the outputs
+glb = postprocessing_utils.to_glb(
+    outputs['gaussian'][0],
+    outputs['mesh'][0],
+    # Optional parameters
+    simplify=0.95,          # Ratio of triangles to remove in the simplification process
+    texture_size=1024,      # Size of the texture used for the GLB
+)
+glb.export("sample.glb")
+
+# Save Gaussians as PLY files
+outputs['gaussian'][0].save_ply("sample.ply")
+```
+
+After running the code, you will get the following files:
+- `sample_gs.mp4`: a video showing the 3D Gaussian representation
+- `sample_rf.mp4`: a video showing the Radiance Field representation
+- `sample_mesh.mp4`: a video showing the mesh representation
+- `sample.glb`: a GLB file containing the extracted textured mesh
+- `sample.ply`: a PLY file containing the 3D Gaussian representation
+
+
+### Web Demo
+
+[app.py](app.py) provides a simple web demo for 3D asset generation. Since this demo is based on [Gradio](https://gradio.app/), additional dependencies are required:
+```sh
+. ./setup.sh --demo
+```
+
+After installing the dependencies, you can run the demo with the following command:
+```sh
+python app.py
+```
+
+Then, you can access the demo at the address shown in the terminal.
+
+***The web demo is also available on [Hugging Face Spaces](https://huggingface.co/spaces/JeffreyXiang/TRELLIS)!***
+
+
+<!-- Dataset -->
+## 📚 Dataset
+
+We provide **TRELLIS-500K**, a large-scale dataset containing 500K 3D assets curated from [Objaverse(XL)](https://objaverse.allenai.org/), [ABO](https://amazon-berkeley-objects.s3.amazonaws.com/index.html), [3D-FUTURE](https://tianchi.aliyun.com/specials/promotion/alibaba-3d-future), [HSSD](https://huggingface.co/datasets/hssd/hssd-models), and [Toys4k](https://github.com/rehg-lab/lowshot-shapebias/tree/main/toys4k), filtered based on aesthetic scores. Please refer to the [dataset README](DATASET.md) for more details.
+
+<!-- License -->
+## ⚖️ License
+
+TRELLIS models and the majority of the code are licensed under the [MIT License](LICENSE). The following submodules may have different licenses:
+- [**diffoctreerast**](https://github.com/JeffreyXiang/diffoctreerast): We developed a CUDA-based real-time differentiable octree renderer for rendering radiance fields as part of this project. This renderer is derived from the [diff-gaussian-rasterization](https://github.com/graphdeco-inria/diff-gaussian-rasterization) project and is available under the [LICENSE](https://github.com/JeffreyXiang/diffoctreerast/blob/master/LICENSE).
+
+
+- [**Modified Flexicubes**](https://github.com/MaxtirError/FlexiCubes): In this project, we used a modified version of [Flexicubes](https://github.com/nv-tlabs/FlexiCubes) to support vertex attributes. This modified version is licensed under the [LICENSE](https://github.com/nv-tlabs/FlexiCubes/blob/main/LICENSE.txt).
+
+
+
+
+<!-- Citation -->
+## 📜 Citation
+
+If you find this work helpful, please consider citing our paper:
+
+```bibtex
+@article{xiang2024structured,
+    title   = {Structured 3D Latents for Scalable and Versatile 3D Generation},
+    author  = {Xiang, Jianfeng and Lv, Zelong and Xu, Sicheng and Deng, Yu and Wang, Ruicheng and Zhang, Bowen and Chen, Dong and Tong, Xin and Yang, Jiaolong},
+    journal = {arXiv preprint arXiv:2412.01506},
+    year    = {2024}
+}
+```
+

TRELLIS/SECURITY.md

@@ -0,0 +1,41 @@
+<!-- BEGIN MICROSOFT SECURITY.MD V0.0.9 BLOCK -->
+
+## Security
+
+Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin).
+
+If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below.
+
+## Reporting Security Issues
+
+**Please do not report security vulnerabilities through public GitHub issues.**
+
+Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report).
+
+If you prefer to submit without logging in, send email to [[email protected]](mailto:[email protected]).  If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp).
+
+You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 
+
+Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
+
+  * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
+  * Full paths of source file(s) related to the manifestation of the issue
+  * The location of the affected source code (tag/branch/commit or direct URL)
+  * Any special configuration required to reproduce the issue
+  * Step-by-step instructions to reproduce the issue
+  * Proof-of-concept or exploit code (if possible)
+  * Impact of the issue, including how an attacker might exploit the issue
+
+This information will help us triage your report more quickly.
+
+If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs.
+
+## Preferred Languages
+
+We prefer all communications to be in English.
+
+## Policy
+
+Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd).
+
+<!-- END MICROSOFT SECURITY.MD BLOCK -->

TRELLIS/SUPPORT.md

@@ -0,0 +1,25 @@
+# TODO: The maintainer of this repo has not yet edited this file
+
+**REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project?
+
+- **No CSS support:** Fill out this template with information about how to file issues and get help.
+- **Yes CSS support:** Fill out an intake form at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). CSS will work with/help you to determine next steps.
+- **Not sure?** Fill out an intake as though the answer were "Yes". CSS will help you decide.
+
+*Then remove this first heading from this SUPPORT.MD file before publishing your repo.*
+
+# Support
+
+## How to file issues and get help  
+
+This project uses GitHub Issues to track bugs and feature requests. Please search the existing 
+issues before filing new issues to avoid duplicates.  For new issues, file your bug or 
+feature request as a new Issue.
+
+For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE 
+FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER
+CHANNEL. WHERE WILL YOU HELP PEOPLE?**.
+
+## Microsoft Support Policy  
+
+Support for this **PROJECT or PRODUCT** is limited to the resources listed above.

TRELLIS/app.py

@@ -0,0 +1,403 @@
+import gradio as gr
+from gradio_litmodel3d import LitModel3D
+
+import os
+import shutil
+from typing import *
+import torch
+import numpy as np
+import imageio
+from easydict import EasyDict as edict
+from PIL import Image
+from trellis.pipelines import TrellisImageTo3DPipeline
+from trellis.representations import Gaussian, MeshExtractResult
+from trellis.utils import render_utils, postprocessing_utils
+
+
+MAX_SEED = np.iinfo(np.int32).max
+TMP_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tmp')
+os.makedirs(TMP_DIR, exist_ok=True)
+
+
+def start_session(req: gr.Request):
+    user_dir = os.path.join(TMP_DIR, str(req.session_hash))
+    os.makedirs(user_dir, exist_ok=True)
+    
+    
+def end_session(req: gr.Request):
+    user_dir = os.path.join(TMP_DIR, str(req.session_hash))
+    shutil.rmtree(user_dir)
+
+
+def preprocess_image(image: Image.Image) -> Image.Image:
+    """
+    Preprocess the input image.
+
+    Args:
+        image (Image.Image): The input image.
+
+    Returns:
+        Image.Image: The preprocessed image.
+    """
+    processed_image = pipeline.preprocess_image(image)
+    return processed_image
+
+
+def preprocess_images(images: List[Tuple[Image.Image, str]]) -> List[Image.Image]:
+    """
+    Preprocess a list of input images.
+    
+    Args:
+        images (List[Tuple[Image.Image, str]]): The input images.
+        
+    Returns:
+        List[Image.Image]: The preprocessed images.
+    """
+    images = [image[0] for image in images]
+    processed_images = [pipeline.preprocess_image(image) for image in images]
+    return processed_images
+
+
+def pack_state(gs: Gaussian, mesh: MeshExtractResult) -> dict:
+    return {
+        'gaussian': {
+            **gs.init_params,
+            '_xyz': gs._xyz.cpu().numpy(),
+            '_features_dc': gs._features_dc.cpu().numpy(),
+            '_scaling': gs._scaling.cpu().numpy(),
+            '_rotation': gs._rotation.cpu().numpy(),
+            '_opacity': gs._opacity.cpu().numpy(),
+        },
+        'mesh': {
+            'vertices': mesh.vertices.cpu().numpy(),
+            'faces': mesh.faces.cpu().numpy(),
+        },
+    }
+    
+    
+def unpack_state(state: dict) -> Tuple[Gaussian, edict, str]:
+    gs = Gaussian(
+        aabb=state['gaussian']['aabb'],
+        sh_degree=state['gaussian']['sh_degree'],
+        mininum_kernel_size=state['gaussian']['mininum_kernel_size'],
+        scaling_bias=state['gaussian']['scaling_bias'],
+        opacity_bias=state['gaussian']['opacity_bias'],
+        scaling_activation=state['gaussian']['scaling_activation'],
+    )
+    gs._xyz = torch.tensor(state['gaussian']['_xyz'], device='cuda')
+    gs._features_dc = torch.tensor(state['gaussian']['_features_dc'], device='cuda')
+    gs._scaling = torch.tensor(state['gaussian']['_scaling'], device='cuda')
+    gs._rotation = torch.tensor(state['gaussian']['_rotation'], device='cuda')
+    gs._opacity = torch.tensor(state['gaussian']['_opacity'], device='cuda')
+    
+    mesh = edict(
+        vertices=torch.tensor(state['mesh']['vertices'], device='cuda'),
+        faces=torch.tensor(state['mesh']['faces'], device='cuda'),
+    )
+    
+    return gs, mesh
+
+
+def get_seed(randomize_seed: bool, seed: int) -> int:
+    """
+    Get the random seed.
+    """
+    return np.random.randint(0, MAX_SEED) if randomize_seed else seed
+
+
+def image_to_3d(
+    image: Image.Image,
+    multiimages: List[Tuple[Image.Image, str]],
+    is_multiimage: bool,
+    seed: int,
+    ss_guidance_strength: float,
+    ss_sampling_steps: int,
+    slat_guidance_strength: float,
+    slat_sampling_steps: int,
+    multiimage_algo: Literal["multidiffusion", "stochastic"],
+    req: gr.Request,
+) -> Tuple[dict, str]:
+    """
+    Convert an image to a 3D model.
+
+    Args:
+        image (Image.Image): The input image.
+        multiimages (List[Tuple[Image.Image, str]]): The input images in multi-image mode.
+        is_multiimage (bool): Whether is in multi-image mode.
+        seed (int): The random seed.
+        ss_guidance_strength (float): The guidance strength for sparse structure generation.
+        ss_sampling_steps (int): The number of sampling steps for sparse structure generation.
+        slat_guidance_strength (float): The guidance strength for structured latent generation.
+        slat_sampling_steps (int): The number of sampling steps for structured latent generation.
+        multiimage_algo (Literal["multidiffusion", "stochastic"]): The algorithm for multi-image generation.
+
+    Returns:
+        dict: The information of the generated 3D model.
+        str: The path to the video of the 3D model.
+    """
+    user_dir = os.path.join(TMP_DIR, str(req.session_hash))
+    if not is_multiimage:
+        outputs = pipeline.run(
+            image,
+            seed=seed,
+            formats=["gaussian", "mesh"],
+            preprocess_image=False,
+            sparse_structure_sampler_params={
+                "steps": ss_sampling_steps,
+                "cfg_strength": ss_guidance_strength,
+            },
+            slat_sampler_params={
+                "steps": slat_sampling_steps,
+                "cfg_strength": slat_guidance_strength,
+            },
+        )
+    else:
+        outputs = pipeline.run_multi_image(
+            [image[0] for image in multiimages],
+            seed=seed,
+            formats=["gaussian", "mesh"],
+            preprocess_image=False,
+            sparse_structure_sampler_params={
+                "steps": ss_sampling_steps,
+                "cfg_strength": ss_guidance_strength,
+            },
+            slat_sampler_params={
+                "steps": slat_sampling_steps,
+                "cfg_strength": slat_guidance_strength,
+            },
+            mode=multiimage_algo,
+        )
+    video = render_utils.render_video(outputs['gaussian'][0], num_frames=120)['color']
+    video_geo = render_utils.render_video(outputs['mesh'][0], num_frames=120)['normal']
+    video = [np.concatenate([video[i], video_geo[i]], axis=1) for i in range(len(video))]
+    video_path = os.path.join(user_dir, 'sample.mp4')
+    imageio.mimsave(video_path, video, fps=15)
+    state = pack_state(outputs['gaussian'][0], outputs['mesh'][0])
+    torch.cuda.empty_cache()
+    return state, video_path
+
+
+def extract_glb(
+    state: dict,
+    mesh_simplify: float,
+    texture_size: int,
+    req: gr.Request,
+) -> Tuple[str, str]:
+    """
+    Extract a GLB file from the 3D model.
+
+    Args:
+        state (dict): The state of the generated 3D model.
+        mesh_simplify (float): The mesh simplification factor.
+        texture_size (int): The texture resolution.
+
+    Returns:
+        str: The path to the extracted GLB file.
+    """
+    user_dir = os.path.join(TMP_DIR, str(req.session_hash))
+    gs, mesh = unpack_state(state)
+    glb = postprocessing_utils.to_glb(gs, mesh, simplify=mesh_simplify, texture_size=texture_size, verbose=False)
+    glb_path = os.path.join(user_dir, 'sample.glb')
+    glb.export(glb_path)
+    torch.cuda.empty_cache()
+    return glb_path, glb_path
+
+
+def extract_gaussian(state: dict, req: gr.Request) -> Tuple[str, str]:
+    """
+    Extract a Gaussian file from the 3D model.
+
+    Args:
+        state (dict): The state of the generated 3D model.
+
+    Returns:
+        str: The path to the extracted Gaussian file.
+    """
+    user_dir = os.path.join(TMP_DIR, str(req.session_hash))
+    gs, _ = unpack_state(state)
+    gaussian_path = os.path.join(user_dir, 'sample.ply')
+    gs.save_ply(gaussian_path)
+    torch.cuda.empty_cache()
+    return gaussian_path, gaussian_path
+
+
+def prepare_multi_example() -> List[Image.Image]:
+    multi_case = list(set([i.split('_')[0] for i in os.listdir("assets/example_multi_image")]))
+    images = []
+    for case in multi_case:
+        _images = []
+        for i in range(1, 4):
+            img = Image.open(f'assets/example_multi_image/{case}_{i}.png')
+            W, H = img.size
+            img = img.resize((int(W / H * 512), 512))
+            _images.append(np.array(img))
+        images.append(Image.fromarray(np.concatenate(_images, axis=1)))
+    return images
+
+
+def split_image(image: Image.Image) -> List[Image.Image]:
+    """
+    Split an image into multiple views.
+    """
+    image = np.array(image)
+    alpha = image[..., 3]
+    alpha = np.any(alpha>0, axis=0)
+    start_pos = np.where(~alpha[:-1] & alpha[1:])[0].tolist()
+    end_pos = np.where(alpha[:-1] & ~alpha[1:])[0].tolist()
+    images = []
+    for s, e in zip(start_pos, end_pos):
+        images.append(Image.fromarray(image[:, s:e+1]))
+    return [preprocess_image(image) for image in images]
+
+
+with gr.Blocks(delete_cache=(600, 600)) as demo:
+    gr.Markdown("""
+    ## Image to 3D Asset with [TRELLIS](https://trellis3d.github.io/)
+    * Upload an image and click "Generate" to create a 3D asset. If the image has alpha channel, it be used as the mask. Otherwise, we use `rembg` to remove the background.
+    * If you find the generated 3D asset satisfactory, click "Extract GLB" to extract the GLB file and download it.
+    """)
+    
+    with gr.Row():
+        with gr.Column():
+            with gr.Tabs() as input_tabs:
+                with gr.Tab(label="Single Image", id=0) as single_image_input_tab:
+                    image_prompt = gr.Image(label="Image Prompt", format="png", image_mode="RGBA", type="pil", height=300)
+                with gr.Tab(label="Multiple Images", id=1) as multiimage_input_tab:
+                    multiimage_prompt = gr.Gallery(label="Image Prompt", format="png", type="pil", height=300, columns=3)
+                    gr.Markdown("""
+                        Input different views of the object in separate images. 
+                        
+                        *NOTE: this is an experimental algorithm without training a specialized model. It may not produce the best results for all images, especially those having different poses or inconsistent details.*
+                    """)
+            
+            with gr.Accordion(label="Generation Settings", open=False):
+                seed = gr.Slider(0, MAX_SEED, label="Seed", value=0, step=1)
+                randomize_seed = gr.Checkbox(label="Randomize Seed", value=True)
+                gr.Markdown("Stage 1: Sparse Structure Generation")
+                with gr.Row():
+                    ss_guidance_strength = gr.Slider(0.0, 10.0, label="Guidance Strength", value=7.5, step=0.1)
+                    ss_sampling_steps = gr.Slider(1, 50, label="Sampling Steps", value=12, step=1)
+                gr.Markdown("Stage 2: Structured Latent Generation")
+                with gr.Row():
+                    slat_guidance_strength = gr.Slider(0.0, 10.0, label="Guidance Strength", value=3.0, step=0.1)
+                    slat_sampling_steps = gr.Slider(1, 50, label="Sampling Steps", value=12, step=1)
+                multiimage_algo = gr.Radio(["stochastic", "multidiffusion"], label="Multi-image Algorithm", value="stochastic")
+
+            generate_btn = gr.Button("Generate")
+            
+            with gr.Accordion(label="GLB Extraction Settings", open=False):
+                mesh_simplify = gr.Slider(0.9, 0.98, label="Simplify", value=0.95, step=0.01)
+                texture_size = gr.Slider(512, 2048, label="Texture Size", value=1024, step=512)
+            
+            with gr.Row():
+                extract_glb_btn = gr.Button("Extract GLB", interactive=False)
+                extract_gs_btn = gr.Button("Extract Gaussian", interactive=False)
+            gr.Markdown("""
+                        *NOTE: Gaussian file can be very large (~50MB), it will take a while to display and download.*
+                        """)
+
+        with gr.Column():
+            video_output = gr.Video(label="Generated 3D Asset", autoplay=True, loop=True, height=300)
+            model_output = LitModel3D(label="Extracted GLB/Gaussian", exposure=10.0, height=300)
+            
+            with gr.Row():
+                download_glb = gr.DownloadButton(label="Download GLB", interactive=False)
+                download_gs = gr.DownloadButton(label="Download Gaussian", interactive=False)  
+    
+    is_multiimage = gr.State(False)
+    output_buf = gr.State()
+
+    # Example images at the bottom of the page
+    with gr.Row() as single_image_example:
+        examples = gr.Examples(
+            examples=[
+                f'assets/example_image/{image}'
+                for image in os.listdir("assets/example_image")
+            ],
+            inputs=[image_prompt],
+            fn=preprocess_image,
+            outputs=[image_prompt],
+            run_on_click=True,
+            examples_per_page=64,
+        )
+    with gr.Row(visible=False) as multiimage_example:
+        examples_multi = gr.Examples(
+            examples=prepare_multi_example(),
+            inputs=[image_prompt],
+            fn=split_image,
+            outputs=[multiimage_prompt],
+            run_on_click=True,
+            examples_per_page=8,
+        )
+
+    # Handlers
+    demo.load(start_session)
+    demo.unload(end_session)
+    
+    single_image_input_tab.select(
+        lambda: tuple([False, gr.Row.update(visible=True), gr.Row.update(visible=False)]),
+        outputs=[is_multiimage, single_image_example, multiimage_example]
+    )
+    multiimage_input_tab.select(
+        lambda: tuple([True, gr.Row.update(visible=False), gr.Row.update(visible=True)]),
+        outputs=[is_multiimage, single_image_example, multiimage_example]
+    )
+    
+    image_prompt.upload(
+        preprocess_image,
+        inputs=[image_prompt],
+        outputs=[image_prompt],
+    )
+    multiimage_prompt.upload(
+        preprocess_images,
+        inputs=[multiimage_prompt],
+        outputs=[multiimage_prompt],
+    )
+
+    generate_btn.click(
+        get_seed,
+        inputs=[randomize_seed, seed],
+        outputs=[seed],
+    ).then(
+        image_to_3d,
+        inputs=[image_prompt, multiimage_prompt, is_multiimage, seed, ss_guidance_strength, ss_sampling_steps, slat_guidance_strength, slat_sampling_steps, multiimage_algo],
+        outputs=[output_buf, video_output],
+    ).then(
+        lambda: tuple([gr.Button(interactive=True), gr.Button(interactive=True)]),
+        outputs=[extract_glb_btn, extract_gs_btn],
+    )
+
+    video_output.clear(
+        lambda: tuple([gr.Button(interactive=False), gr.Button(interactive=False)]),
+        outputs=[extract_glb_btn, extract_gs_btn],
+    )
+
+    extract_glb_btn.click(
+        extract_glb,
+        inputs=[output_buf, mesh_simplify, texture_size],
+        outputs=[model_output, download_glb],
+    ).then(
+        lambda: gr.Button(interactive=True),
+        outputs=[download_glb],
+    )
+    
+    extract_gs_btn.click(
+        extract_gaussian,
+        inputs=[output_buf],
+        outputs=[model_output, download_gs],
+    ).then(
+        lambda: gr.Button(interactive=True),
+        outputs=[download_gs],
+    )
+
+    model_output.clear(
+        lambda: gr.Button(interactive=False),
+        outputs=[download_glb],
+    )
+    
+
+# Launch the Gradio app
+if __name__ == "__main__":
+    pipeline = TrellisImageTo3DPipeline.from_pretrained("JeffreyXiang/TRELLIS-image-large")
+    pipeline.cuda()
+    demo.launch()