Upload folder using huggingface_hub

.ipynb_checkpoints/config-checkpoint.json

@@ -0,0 +1,13 @@
+{
+  "model_name": "Paris",
+  "num_experts": 8,
+  "experiment_name": "dit_xl2_multi_expert_pretrained_text",
+  "dataset_name": "LAION-Aesthetic",
+  "image_size": 32,
+  "num_channels": 4,
+  "expert_architecture": "dit",
+  "router_architecture": "dit",
+  "use_latents": true,
+  "vae_name": "stabilityai/sd-vae-ft-mse",
+  "text_encoder_name": "openai/clip-vit-large-patch14"
+}

.ipynb_checkpoints/model_index-checkpoint.json

@@ -0,0 +1,21 @@
+{
+  "_class_name": "DDMPipeline",
+  "_diffusers_version": "0.21.0",
+  "num_experts": 8,
+  "expert_architecture": "dit",
+  "router_architecture": "dit",
+  "vae": [
+    "diffusers",
+    "AutoencoderKL"
+  ],
+  "text_encoder": [
+    "transformers",
+    "CLIPTextModel"
+  ],
+  "tokenizer": [
+    "transformers",
+    "CLIPTokenizer"
+  ],
+  "vae_name": "stabilityai/sd-vae-ft-mse",
+  "text_encoder_name": "openai/clip-vit-large-patch14"
+}

README.md

@@ -1,13 +1,113 @@
 ---
 license: mit
-language:
-- en
-pipeline_tag: text-to-image
 tags:
-- diffusion
-- decentralized
-- distributed
-- distributed-training
-- image-generation
 - text-to-image
----
+- diffusion
+- multi-expert
+- dit
+- laion
+---
+
+# Paris
+
+A multi-expert diffusion model trained with dynamic expert routing on LAION-Aesthetic.
+
+## Model Description
+
+This model uses **8 specialized DiT experts** with a learned router that dynamically selects the best expert for each generation based on the noisy latent and timestep.
+
+- **Architecture**: dit-XL/2 with 8 experts
+- **Router**: dit-based routing network
+- **Hidden Size**: 1152
+- **Layers**: 28
+- **Attention Heads**: 16
+- **Parameters per Expert**: ~0M
+- **Total Parameters**: ~3M
+- **Text Conditioning**: ✓ (CLIP ViT-L/14)
+- **Training Dataset**: LAION-Aesthetic
+
+## Model Structure
+
+```
+Paris/
+├── config.json              # High-level model configuration
+├── model_index.json         # Pipeline component index
+├── expert_0/                # Specialized expert models
+│   ├── config.json
+│   └── diffusion_pytorch_model.safetensors
+├── expert_1/ ... expert_7/
+├── router/                  # Dynamic routing network
+│   ├── config.json
+│   └── pytorch_model.safetensors
+├── vae/                     # VAE (sd-vae-ft-mse)
+├── text_encoder/            # CLIP text encoder
+├── tokenizer/               # CLIP tokenizer
+└── inference_pipeline.py    # Custom inference code
+```
+
+## Usage
+
+```python
+from inference_pipeline import DDMPipeline
+
+# Load the pipeline
+pipeline = DDMPipeline.from_pretrained("paris")
+
+# Generate images
+images = pipeline(
+    prompt="A beautiful sunset over Paris, oil painting style",
+    num_inference_steps=50,
+    guidance_scale=7.5,
+    num_images=4
+)
+
+# Save images
+for i, img in enumerate(images):
+    img.save(f"output_{i}.png")
+```
+
+## Training Details
+
+- **Base Model**: DiT-XL/2 pretrained on ImageNet
+- **Batch Size**: 16 per expert
+- **Learning Rate**: 2e-05
+- **Image Size**: 256x256 (32x32 latent space)
+- **VAE**: SD VAE (8x downsampling)
+- **Text Encoder**: CLIP ViT-L/14
+- **EMA**: True
+- **Mixed Precision**: True
+
+### Multi-Expert Architecture
+
+Each expert specializes in different visual styles/content through dynamic routing:
+- The router network analyzes the noisy latent and timestep
+- Selects the most appropriate expert for denoising
+- Enables better quality and diversity compared to single models
+
+
+
+## Examples
+
+Coming soon! Check back for generated examples.
+
+## Limitations
+
+- Trained on LAION-Aesthetic which may contain biases
+- Best results at 256x256 resolution
+- Requires GPU for inference (8GB+ VRAM recommended)
+
+## Citation
+
+```bibtex
+@misc{paris,
+  author = {Your Name},
+  title = {Paris: Multi-Expert Diffusion Model},
+  year = {2024},
+  publisher = {HuggingFace},
+  url = {https://huggingface.co/paris}
+}
+```
+
+## License
+
+MIT License

config.json

@@ -0,0 +1,13 @@
+{
+  "model_name": "Paris",
+  "num_experts": 8,
+  "experiment_name": "dit_xl2_multi_expert_pretrained_text",
+  "dataset_name": "LAION-Aesthetic",
+  "image_size": 32,
+  "num_channels": 4,
+  "expert_architecture": "dit",
+  "router_architecture": "dit",
+  "use_latents": true,
+  "vae_name": "stabilityai/sd-vae-ft-mse",
+  "text_encoder_name": "openai/clip-vit-large-patch14"
+}

expert_0/.ipynb_checkpoints/config-checkpoint.json

@@ -0,0 +1,19 @@
+{
+  "model_type": "dit_expert",
+  "expert_id": 0,
+  "architecture": "dit",
+  "hidden_size": 1152,
+  "num_layers": 28,
+  "num_heads": 16,
+  "patch_size": 2,
+  "in_channels": 4,
+  "out_channels": 4,
+  "mlp_ratio": 4.0,
+  "use_text_conditioning": true,
+  "use_class_conditioning": false,
+  "num_classes": 1000,
+  "text_embed_dim": 768,
+  "use_dit_time_embed": true,
+  "use_adaln_single": true,
+  "cfg_dropout_prob": 0.1
+}

expert_0/.ipynb_checkpoints/model_index-checkpoint.json

@@ -0,0 +1,5 @@
+{
+  "_class_name": "DiTExpert",
+  "_diffusers_version": "0.21.0",
+  "expert_id": 0
+}

expert_0/config.json

@@ -0,0 +1,19 @@
+{
+  "model_type": "dit_expert",
+  "expert_id": 0,
+  "architecture": "dit",
+  "hidden_size": 1152,
+  "num_layers": 28,
+  "num_heads": 16,
+  "patch_size": 2,
+  "in_channels": 4,
+  "out_channels": 4,
+  "mlp_ratio": 4.0,
+  "use_text_conditioning": true,
+  "use_class_conditioning": false,
+  "num_classes": 1000,
+  "text_embed_dim": 768,
+  "use_dit_time_embed": true,
+  "use_adaln_single": true,
+  "cfg_dropout_prob": 0.1
+}

expert_0/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a10067fe64cc3ee3137d5c2c88fdc6ca8dea420a5875131969c11b4a4cf24ba
+size 2423920240

expert_0/model_index.json

@@ -0,0 +1,5 @@
+{
+  "_class_name": "DiTExpert",
+  "_diffusers_version": "0.21.0",
+  "expert_id": 0
+}

expert_1/config.json

@@ -0,0 +1,19 @@
+{
+  "model_type": "dit_expert",
+  "expert_id": 1,
+  "architecture": "dit",
+  "hidden_size": 1152,
+  "num_layers": 28,
+  "num_heads": 16,
+  "patch_size": 2,
+  "in_channels": 4,
+  "out_channels": 4,
+  "mlp_ratio": 4.0,
+  "use_text_conditioning": true,
+  "use_class_conditioning": false,
+  "num_classes": 1000,
+  "text_embed_dim": 768,
+  "use_dit_time_embed": true,
+  "use_adaln_single": true,
+  "cfg_dropout_prob": 0.1
+}

expert_1/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:880e77a2924166cae2f533b7e67616f5fec4db2c912c1e68b7baf2a12e529901
+size 2423920240

expert_1/model_index.json

@@ -0,0 +1,5 @@
+{
+  "_class_name": "DiTExpert",
+  "_diffusers_version": "0.21.0",
+  "expert_id": 1
+}

expert_2/config.json

@@ -0,0 +1,19 @@
+{
+  "model_type": "dit_expert",
+  "expert_id": 2,
+  "architecture": "dit",
+  "hidden_size": 1152,
+  "num_layers": 28,
+  "num_heads": 16,
+  "patch_size": 2,
+  "in_channels": 4,
+  "out_channels": 4,
+  "mlp_ratio": 4.0,
+  "use_text_conditioning": true,
+  "use_class_conditioning": false,
+  "num_classes": 1000,
+  "text_embed_dim": 768,
+  "use_dit_time_embed": true,
+  "use_adaln_single": true,
+  "cfg_dropout_prob": 0.1
+}

expert_2/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c481af35025e018eb489087260afe968d2e530614c6cfd52d9e17143975936b0
+size 2423920240

expert_2/model_index.json

@@ -0,0 +1,5 @@
+{
+  "_class_name": "DiTExpert",
+  "_diffusers_version": "0.21.0",
+  "expert_id": 2
+}

expert_3/config.json

@@ -0,0 +1,19 @@
+{
+  "model_type": "dit_expert",
+  "expert_id": 3,
+  "architecture": "dit",
+  "hidden_size": 1152,
+  "num_layers": 28,
+  "num_heads": 16,
+  "patch_size": 2,
+  "in_channels": 4,
+  "out_channels": 4,
+  "mlp_ratio": 4.0,
+  "use_text_conditioning": true,
+  "use_class_conditioning": false,
+  "num_classes": 1000,
+  "text_embed_dim": 768,
+  "use_dit_time_embed": true,
+  "use_adaln_single": true,
+  "cfg_dropout_prob": 0.1
+}

expert_3/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e88eab8b78086c113cc6a2f300ceaa1753be899b1abbf4bce115f4b86b2324d4
+size 2423920240

expert_3/model_index.json

@@ -0,0 +1,5 @@
+{
+  "_class_name": "DiTExpert",
+  "_diffusers_version": "0.21.0",
+  "expert_id": 3
+}

expert_4/config.json

@@ -0,0 +1,19 @@
+{
+  "model_type": "dit_expert",
+  "expert_id": 4,
+  "architecture": "dit",
+  "hidden_size": 1152,
+  "num_layers": 28,
+  "num_heads": 16,
+  "patch_size": 2,
+  "in_channels": 4,
+  "out_channels": 4,
+  "mlp_ratio": 4.0,
+  "use_text_conditioning": true,
+  "use_class_conditioning": false,
+  "num_classes": 1000,
+  "text_embed_dim": 768,
+  "use_dit_time_embed": true,
+  "use_adaln_single": true,
+  "cfg_dropout_prob": 0.1
+}

expert_4/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6d57fa16c5bb6c4941d4bada172607926edc23c5ac01d5fffdba0819426748f
+size 2423920240

expert_4/model_index.json

@@ -0,0 +1,5 @@
+{
+  "_class_name": "DiTExpert",
+  "_diffusers_version": "0.21.0",
+  "expert_id": 4
+}

expert_5/config.json

@@ -0,0 +1,19 @@
+{
+  "model_type": "dit_expert",
+  "expert_id": 5,
+  "architecture": "dit",
+  "hidden_size": 1152,
+  "num_layers": 28,
+  "num_heads": 16,
+  "patch_size": 2,
+  "in_channels": 4,
+  "out_channels": 4,
+  "mlp_ratio": 4.0,
+  "use_text_conditioning": true,
+  "use_class_conditioning": false,
+  "num_classes": 1000,
+  "text_embed_dim": 768,
+  "use_dit_time_embed": true,
+  "use_adaln_single": true,
+  "cfg_dropout_prob": 0.1
+}

expert_5/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:189dc4afa4fa8bd5b99487d7ce095b1918f4d30bc2e20a21df2273d4280858dc
+size 2423920240

expert_5/model_index.json

@@ -0,0 +1,5 @@
+{
+  "_class_name": "DiTExpert",
+  "_diffusers_version": "0.21.0",
+  "expert_id": 5
+}

expert_6/config.json

@@ -0,0 +1,19 @@
+{
+  "model_type": "dit_expert",
+  "expert_id": 6,
+  "architecture": "dit",
+  "hidden_size": 1152,
+  "num_layers": 28,
+  "num_heads": 16,
+  "patch_size": 2,
+  "in_channels": 4,
+  "out_channels": 4,
+  "mlp_ratio": 4.0,
+  "use_text_conditioning": true,
+  "use_class_conditioning": false,
+  "num_classes": 1000,
+  "text_embed_dim": 768,
+  "use_dit_time_embed": true,
+  "use_adaln_single": true,
+  "cfg_dropout_prob": 0.1
+}

expert_6/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1be832b1326ad751bc18266f6fbd712b29ec4b6b6dccadc614b29d9627283b5e
+size 2423920240

expert_6/model_index.json

@@ -0,0 +1,5 @@
+{
+  "_class_name": "DiTExpert",
+  "_diffusers_version": "0.21.0",
+  "expert_id": 6
+}

expert_7/config.json

@@ -0,0 +1,19 @@
+{
+  "model_type": "dit_expert",
+  "expert_id": 7,
+  "architecture": "dit",
+  "hidden_size": 1152,
+  "num_layers": 28,
+  "num_heads": 16,
+  "patch_size": 2,
+  "in_channels": 4,
+  "out_channels": 4,
+  "mlp_ratio": 4.0,
+  "use_text_conditioning": true,
+  "use_class_conditioning": false,
+  "num_classes": 1000,
+  "text_embed_dim": 768,
+  "use_dit_time_embed": true,
+  "use_adaln_single": true,
+  "cfg_dropout_prob": 0.1
+}

expert_7/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef7de12263d23f503624336bed6dde4da77140b0137f23405799d8a9951b4b50
+size 2423920240

expert_7/model_index.json

@@ -0,0 +1,5 @@
+{
+  "_class_name": "DiTExpert",
+  "_diffusers_version": "0.21.0",
+  "expert_id": 7
+}

model_index.json

@@ -0,0 +1,21 @@
+{
+  "_class_name": "DDMPipeline",
+  "_diffusers_version": "0.21.0",
+  "num_experts": 8,
+  "expert_architecture": "dit",
+  "router_architecture": "dit",
+  "vae": [
+    "diffusers",
+    "AutoencoderKL"
+  ],
+  "text_encoder": [
+    "transformers",
+    "CLIPTextModel"
+  ],
+  "tokenizer": [
+    "transformers",
+    "CLIPTokenizer"
+  ],
+  "vae_name": "stabilityai/sd-vae-ft-mse",
+  "text_encoder_name": "openai/clip-vit-large-patch14"
+}

router/.ipynb_checkpoints/config-checkpoint.json

@@ -0,0 +1,11 @@
+{
+  "model_type": "router",
+  "architecture": "dit",
+  "num_clusters": 8,
+  "hidden_size": 768,
+  "num_layers": 12,
+  "num_heads": 12,
+  "patch_size": 2,
+  "in_channels": 4,
+  "use_dit_time_embed": false
+}

router/config.json

@@ -0,0 +1,11 @@
+{
+  "model_type": "router",
+  "architecture": "dit",
+  "num_clusters": 8,
+  "hidden_size": 768,
+  "num_layers": 12,
+  "num_heads": 12,
+  "patch_size": 2,
+  "in_channels": 4,
+  "use_dit_time_embed": true
+}

router/model_index.json

@@ -0,0 +1,4 @@
+{
+  "_class_name": "Router",
+  "_diffusers_version": "0.21.0"
+}

router/pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b819e8d9c0b25d174703d799a335bae2631ab849925b33c9d9a8dcd8b426b80
+size 516560576

text_encoder/config.json

@@ -0,0 +1,24 @@
+{
+  "architectures": [
+    "CLIPTextModel"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dropout": 0.0,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "hidden_act": "quick_gelu",
+  "hidden_size": 768,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 77,
+  "model_type": "clip_text_model",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "projection_dim": 768,
+  "transformers_version": "4.56.2",
+  "vocab_size": 49408
+}

text_encoder/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:778d02eb9e707c3fbaae0b67b79ea0d1399b52e624fb634f2f19375ae7c047c3
+size 492265168

tokenizer/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer/tokenizer_config.json

@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "49406": {
+      "content": "<|startoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49407": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "do_lower_case": true,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 77,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "CLIPTokenizer",
+  "unk_token": "<|endoftext|>"
+}

vae/config.json

@@ -0,0 +1,38 @@
+{
+  "_class_name": "AutoencoderKL",
+  "_diffusers_version": "0.35.1",
+  "_name_or_path": "stabilityai/sd-vae-ft-mse",
+  "act_fn": "silu",
+  "block_out_channels": [
+    128,
+    256,
+    512,
+    512
+  ],
+  "down_block_types": [
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D"
+  ],
+  "force_upcast": true,
+  "in_channels": 3,
+  "latent_channels": 4,
+  "latents_mean": null,
+  "latents_std": null,
+  "layers_per_block": 2,
+  "mid_block_add_attention": true,
+  "norm_num_groups": 32,
+  "out_channels": 3,
+  "sample_size": 256,
+  "scaling_factor": 0.18215,
+  "shift_factor": null,
+  "up_block_types": [
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D"
+  ],
+  "use_post_quant_conv": true,
+  "use_quant_conv": true
+}

vae/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2aa1f43011b553a4cba7f37456465cdbd48aab7b54b9348b890e8058ea7683ec
+size 334643268