feat: controlnet example

.env.example

@@ -1,4 +1,5 @@
 COMFY_API_URL="http://127.0.0.1:3000"
 COMFY_API_TOKEN=""
 COMFY_DEPLOYMENT_ID=""
-COMFY_DEPLOYMENT_ID_IMG_2_IMG=""
+COMFY_DEPLOYMENT_ID_IMG_2_IMG=""
+COMFY_DEPLOYMENT_ID_CONTROLNET=""

package.json

@@ -11,6 +11,8 @@
   "dependencies": {
     "@hookform/resolvers": "^3.3.4",
     "@radix-ui/react-label": "^2.0.2",
+    "@radix-ui/react-select": "^2.0.0",
+    "@radix-ui/react-separator": "^1.0.3",
     "@radix-ui/react-slot": "^1.0.2",
     "@radix-ui/react-tabs": "^1.0.4",
     "class-variance-authority": "^0.7.0",

src/app/page.tsx

@@ -10,19 +10,32 @@ import {
   checkStatus,
   generate,
   generate_img,
+  generate_img_with_controlnet,
   getUploadUrl,
 } from "@/server/generate";
 import { useEffect, useState } from "react";
 
+import {
+  Select,
+  SelectContent,
+  SelectGroup,
+  SelectItem,
+  SelectLabel,
+  SelectTrigger,
+  SelectValue,
+} from "@/components/ui/select";
+import { Separator } from "@/components/ui/separator";
+
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 
 export default function Page() {
   return (
     <main className="flex min-h-screen flex-col items-center justify-between mt-10">
-      <Tabs defaultValue="txt2img" className="w-full max-w-[500px]">
-        <TabsList className="grid w-full grid-cols-2">
+      <Tabs defaultValue="txt2img" className="w-full max-w-[600px]">
+        <TabsList className="grid w-full grid-cols-3">
           <TabsTrigger value="txt2img">txt2img</TabsTrigger>
           <TabsTrigger value="img2img">img2img</TabsTrigger>
+          <TabsTrigger value="controlpose">Controlpose</TabsTrigger>
         </TabsList>
         <TabsContent value="txt2img">
           <Txt2img />
@@ -30,6 +43,9 @@ export default function Page() {
         <TabsContent value="img2img">
           <Img2img />
         </TabsContent>
+        <TabsContent value="controlpose">
+          <OpenposeToImage />
+        </TabsContent>
       </Tabs>
     </main>
   );
@@ -60,7 +76,7 @@ function Txt2img() {
   }, [runId]);
 
   return (
-    <Card className="w-full max-w-[500px]">
+    <Card className="w-full max-w-[600px]">
       <CardHeader>
         Comfy Deploy - Vector Line Art Tool
         <div className="text-xs text-foreground opacity-50">
@@ -107,7 +123,8 @@ function Txt2img() {
                 className="w-full h-full object-contain"
                 src={image}
                 alt="Generated image"
-              ></img>
+              >
+              </img>
             )}
             {!image && status && (
               <div className="absolute top-0 left-0 w-full h-full flex items-center justify-center gap-2">
@@ -152,7 +169,7 @@ function Img2img() {
   }, [runId]);
 
   return (
-    <Card className="w-full max-w-[500px]">
+    <Card className="w-full max-w-[600px]">
       <CardHeader>Comfy Deploy - Scribble to Anime Girl</CardHeader>
       <CardContent>
         <form
@@ -215,7 +232,8 @@ function Img2img() {
                 className="w-full h-full object-contain"
                 src={image}
                 alt="Generated image"
-              ></img>
+              >
+              </img>
             )}
             {!image && status && (
               <div className="absolute top-0 left-0 w-full h-full flex items-center justify-center gap-2">
@@ -229,3 +247,173 @@ function Img2img() {
     </Card>
   );
 }
+
+const poses = {
+  arms_on_hips: {
+    url:
+      "https://pub-6230db03dc3a4861a9c3e55145ceda44.r2.dev/openpose-pose%20(1).png",
+    name: "Arms on Hips",
+  },
+  waving: {
+    url:
+      "https://pub-6230db03dc3a4861a9c3e55145ceda44.r2.dev/openpose-pose%20(2).png",
+    name: "Waving",
+  },
+  legs_together_sideways: {
+    url:
+      "https://pub-6230db03dc3a4861a9c3e55145ceda44.r2.dev/openpose-pose%20(3).png",
+    name: "Legs together, body at an angle",
+  },
+  excited_jump: {
+    url:
+      "https://pub-6230db03dc3a4861a9c3e55145ceda44.r2.dev/openpose-pose%20(4).png",
+    name: "excited jump",
+  },
+  pointing_to_the_stars: {
+    url:
+      "https://pub-6230db03dc3a4861a9c3e55145ceda44.r2.dev/openpose-pose%20(5).png",
+    name: "Pointing to the stars",
+  },
+};
+
+function OpenposeToImage() {
+  const [prompt, setPrompt] = useState("");
+  const [poseImageUrl, setPoseImageUrl] = useState(
+    "https://pub-6230db03dc3a4861a9c3e55145ceda44.r2.dev/openpose-pose%20(1).png",
+  );
+  const [poseLoading, setPoseLoading] = useState(false);
+  const [image, setImage] = useState("");
+  const [loading, setLoading] = useState(false);
+  const [runId, setRunId] = useState("");
+  const [status, setStatus] = useState<string>();
+
+  const handleSelectChange = (value: keyof typeof poses) => {
+    setPoseImageUrl(poses[value].url); // Update image based on selection
+  };
+
+  // Polling in frontend to check for the
+  useEffect(() => {
+    if (!runId) return;
+    const interval = setInterval(() => {
+      checkStatus(runId).then((res) => {
+        if (res) setStatus(res.status);
+        if (res && res.status === "success") {
+          console.log(res.outputs[0]?.data);
+          setImage(res.outputs[0]?.data?.images[0].url);
+          setLoading(false);
+          clearInterval(interval);
+        }
+      });
+    }, 2000);
+    return () => clearInterval(interval);
+  }, [runId]);
+
+  return (
+    <Card className="w-full max-w-[600px]">
+      <CardHeader>
+        Comfy Deploy - Pose Creator Tool
+        <div className="text-xs text-foreground opacity-50">
+          OpenPose -{" "}
+          <a href="https://civitai.com/models/13647/super-pose-book-vol1-controlnet">
+            pose book
+          </a>
+        </div>
+      </CardHeader>
+      <CardContent>
+        <form
+          className="grid w-full items-center gap-1.5"
+          onSubmit={(e) => {
+            if (loading) return;
+
+            e.preventDefault();
+            setLoading(true);
+            generate_img_with_controlnet(poseImageUrl, prompt).then((res) => {
+              console.log('here', res);
+              if (!res) {
+                setStatus("error");
+                setLoading(false);
+                return;
+              }
+              setRunId(res.run_id);
+            });
+            setStatus("preparing");
+          }}
+        >
+          <Select
+            defaultValue={"Arms on Hips"}
+            onValueChange={(value) => {
+              handleSelectChange(value as keyof typeof poses);
+              setPoseLoading(true); // Start loading when a new pose is selected
+            }}
+          >
+            <Label htmlFor="picture">Pose</Label>
+            <SelectTrigger className="w-[180px]">
+              <SelectValue placeholder="Select a Pose" />
+            </SelectTrigger>
+            <SelectContent>
+              <SelectGroup>
+                <SelectLabel>Poses</SelectLabel>
+                {Object.entries(poses).map(([poseName, attr]) => (
+                  <SelectItem key={poseName} value={poseName}>
+                    {attr.name}
+                  </SelectItem>
+                ))}
+              </SelectGroup>
+            </SelectContent>
+          </Select>
+          <Label htmlFor="picture">Image prompt</Label>
+          <Input
+            id="picture"
+            type="text"
+            value={prompt}
+            onChange={(e) => setPrompt(e.target.value)}
+          />
+          <Button type="submit" className="flex gap-2" disabled={loading}>
+            Generate {loading && <LoadingIcon />}
+          </Button>
+
+          <div className="flex gap-4">
+            <div className="w-full rounded-lg relative">
+              {/* Pose Image */}
+              {poseLoading && (
+                <div className="absolute top-0 left-0 w-full h-full flex items-center justify-center">
+                  <LoadingIcon />
+                </div>
+              )}
+              {poseImageUrl && (
+                <img
+                  className="w-full h-full object-contain"
+                  src={poseImageUrl}
+                  alt="Selected pose"
+                  onLoad={() => setPoseLoading(false)}
+                >
+                </img>
+              )}
+            </div>
+            <Separator
+              orientation="vertical"
+              className="border-gray-200"
+              decorative
+            />
+            <div className="border border-gray-200 w-full square h-[400px] rounded-lg relative">
+              {!loading && image && (
+                <img
+                  className="w-full h-full object-contain"
+                  src={image}
+                  alt="Generated image"
+                >
+                </img>
+              )}
+              {!image && status && (
+                <div className="absolute top-0 left-0 w-full h-full flex items-center justify-center gap-2">
+                  {status} <LoadingIcon />
+                </div>
+              )}
+              {loading && <Skeleton className="w-full h-full" />}
+            </div>
+          </div>
+        </form>
+      </CardContent>
+    </Card>
+  );
+}

src/components/ui/select.tsx

@@ -0,0 +1,160 @@
+"use client"
+
+import * as React from "react"
+import * as SelectPrimitive from "@radix-ui/react-select"
+import { Check, ChevronDown, ChevronUp } from "lucide-react"
+
+import { cn } from "@/lib/utils"
+
+const Select = SelectPrimitive.Root
+
+const SelectGroup = SelectPrimitive.Group
+
+const SelectValue = SelectPrimitive.Value
+
+const SelectTrigger = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Trigger>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Trigger>
+>(({ className, children, ...props }, ref) => (
+  <SelectPrimitive.Trigger
+    ref={ref}
+    className={cn(
+      "flex h-10 w-full items-center justify-between rounded-md border border-input bg-background px-3 py-2 text-sm ring-offset-background placeholder:text-muted-foreground focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 [&>span]:line-clamp-1",
+      className
+    )}
+    {...props}
+  >
+    {children}
+    <SelectPrimitive.Icon asChild>
+      <ChevronDown className="h-4 w-4 opacity-50" />
+    </SelectPrimitive.Icon>
+  </SelectPrimitive.Trigger>
+))
+SelectTrigger.displayName = SelectPrimitive.Trigger.displayName
+
+const SelectScrollUpButton = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.ScrollUpButton>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.ScrollUpButton>
+>(({ className, ...props }, ref) => (
+  <SelectPrimitive.ScrollUpButton
+    ref={ref}
+    className={cn(
+      "flex cursor-default items-center justify-center py-1",
+      className
+    )}
+    {...props}
+  >
+    <ChevronUp className="h-4 w-4" />
+  </SelectPrimitive.ScrollUpButton>
+))
+SelectScrollUpButton.displayName = SelectPrimitive.ScrollUpButton.displayName
+
+const SelectScrollDownButton = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.ScrollDownButton>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.ScrollDownButton>
+>(({ className, ...props }, ref) => (
+  <SelectPrimitive.ScrollDownButton
+    ref={ref}
+    className={cn(
+      "flex cursor-default items-center justify-center py-1",
+      className
+    )}
+    {...props}
+  >
+    <ChevronDown className="h-4 w-4" />
+  </SelectPrimitive.ScrollDownButton>
+))
+SelectScrollDownButton.displayName =
+  SelectPrimitive.ScrollDownButton.displayName
+
+const SelectContent = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Content>
+>(({ className, children, position = "popper", ...props }, ref) => (
+  <SelectPrimitive.Portal>
+    <SelectPrimitive.Content
+      ref={ref}
+      className={cn(
+        "relative z-50 max-h-96 min-w-[8rem] overflow-hidden rounded-md border bg-popover text-popover-foreground shadow-md data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2",
+        position === "popper" &&
+          "data-[side=bottom]:translate-y-1 data-[side=left]:-translate-x-1 data-[side=right]:translate-x-1 data-[side=top]:-translate-y-1",
+        className
+      )}
+      position={position}
+      {...props}
+    >
+      <SelectScrollUpButton />
+      <SelectPrimitive.Viewport
+        className={cn(
+          "p-1",
+          position === "popper" &&
+            "h-[var(--radix-select-trigger-height)] w-full min-w-[var(--radix-select-trigger-width)]"
+        )}
+      >
+        {children}
+      </SelectPrimitive.Viewport>
+      <SelectScrollDownButton />
+    </SelectPrimitive.Content>
+  </SelectPrimitive.Portal>
+))
+SelectContent.displayName = SelectPrimitive.Content.displayName
+
+const SelectLabel = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Label>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Label>
+>(({ className, ...props }, ref) => (
+  <SelectPrimitive.Label
+    ref={ref}
+    className={cn("py-1.5 pl-8 pr-2 text-sm font-semibold", className)}
+    {...props}
+  />
+))
+SelectLabel.displayName = SelectPrimitive.Label.displayName
+
+const SelectItem = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Item>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Item>
+>(({ className, children, ...props }, ref) => (
+  <SelectPrimitive.Item
+    ref={ref}
+    className={cn(
+      "relative flex w-full cursor-default select-none items-center rounded-sm py-1.5 pl-8 pr-2 text-sm outline-none focus:bg-accent focus:text-accent-foreground data-[disabled]:pointer-events-none data-[disabled]:opacity-50",
+      className
+    )}
+    {...props}
+  >
+    <span className="absolute left-2 flex h-3.5 w-3.5 items-center justify-center">
+      <SelectPrimitive.ItemIndicator>
+        <Check className="h-4 w-4" />
+      </SelectPrimitive.ItemIndicator>
+    </span>
+
+    <SelectPrimitive.ItemText>{children}</SelectPrimitive.ItemText>
+  </SelectPrimitive.Item>
+))
+SelectItem.displayName = SelectPrimitive.Item.displayName
+
+const SelectSeparator = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Separator>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Separator>
+>(({ className, ...props }, ref) => (
+  <SelectPrimitive.Separator
+    ref={ref}
+    className={cn("-mx-1 my-1 h-px bg-muted", className)}
+    {...props}
+  />
+))
+SelectSeparator.displayName = SelectPrimitive.Separator.displayName
+
+export {
+  Select,
+  SelectGroup,
+  SelectValue,
+  SelectTrigger,
+  SelectContent,
+  SelectLabel,
+  SelectItem,
+  SelectSeparator,
+  SelectScrollUpButton,
+  SelectScrollDownButton,
+}

src/components/ui/separator.tsx

@@ -0,0 +1,31 @@
+"use client"
+
+import * as React from "react"
+import * as SeparatorPrimitive from "@radix-ui/react-separator"
+
+import { cn } from "@/lib/utils"
+
+const Separator = React.forwardRef<
+  React.ElementRef<typeof SeparatorPrimitive.Root>,
+  React.ComponentPropsWithoutRef<typeof SeparatorPrimitive.Root>
+>(
+  (
+    { className, orientation = "horizontal", decorative = true, ...props },
+    ref
+  ) => (
+    <SeparatorPrimitive.Root
+      ref={ref}
+      decorative={decorative}
+      orientation={orientation}
+      className={cn(
+        "shrink-0 bg-border",
+        orientation === "horizontal" ? "h-[1px] w-full" : "h-full w-[1px]",
+        className
+      )}
+      {...props}
+    />
+  )
+)
+Separator.displayName = SeparatorPrimitive.Root.displayName
+
+export { Separator }

src/lib/comfy-deploy.ts

@@ -6,11 +6,11 @@ const runTypes = z.object({
 
 const runOutputTypes = z.object({
   id: z.string(),
-  status: z.enum(["success", "failed", "running", "uploading"]),
+  status: z.enum(["success", "failed", "running", "uploading", "not-started"]),
   outputs: z.array(
     z.object({
       data: z.any(),
-    })
+    }),
   ),
 });
 
@@ -18,15 +18,16 @@ const uploadFileTypes = z.object({
   upload_url: z.string(),
   file_id: z.string(),
   download_url: z.string(),
-})
+});
 
 export class ComfyDeployClient {
   apiBase: string = "https://www.comfydeploy.com/api";
   apiToken: string;
 
   constructor({ apiBase, apiToken }: { apiBase?: string; apiToken: string }) {
-    if (apiBase)
+    if (apiBase) {
       this.apiBase = `${apiBase}/api`;
+    }
     this.apiToken = apiToken;
   }
 
@@ -47,12 +48,16 @@ export class ComfyDeployClient {
         deployment_id: deployment_id,
         inputs: inputs,
       }),
-      cache: "no-store"
+      cache: "no-store",
     })
-      .then((response) => response.json())
-      .then((json) => runTypes.parse(json))
+      .then((response) => {
+        console.log('response', response)
+        return response.json()})
+      .then((json) => {
+        console.log('json', json)
+        return runTypes.parse(json)})
       .catch((err) => {
-        console.error(err);
+        console.error('err', err);
         return null;
       });
   }
@@ -64,7 +69,7 @@ export class ComfyDeployClient {
         "Content-Type": "application/json",
         authorization: `Bearer ${this.apiToken}`,
       },
-      cache: "no-store"
+      cache: "no-store",
     })
       .then((response) => response.json())
       .then((json) => runOutputTypes.parse(json))
@@ -110,13 +115,13 @@ export class ComfyDeployClient {
     };
     const url = new URL(`${this.apiBase}/upload-url`);
     url.search = new URLSearchParams(obj).toString();
-    
+
     return await fetch(url.href, {
       method: "GET",
       headers: {
         authorization: `Bearer ${this.apiToken}`,
       },
-      cache: "no-store"
+      cache: "no-store",
     })
       .then((response) => response.json())
       .then((json) => uploadFileTypes.parse(json))
@@ -125,4 +130,4 @@ export class ComfyDeployClient {
         return null;
       });
   }
-}
+}

src/server/generate.tsx

@@ -25,6 +25,16 @@ export async function generate_img(input_image: string){
     })
 }
 
+export async function generate_img_with_controlnet(input_openpose_url: string, prompt: string){
+    return await client.run({
+        deployment_id: process.env.COMFY_DEPLOYMENT_ID_CONTROLNET!,
+        inputs: {
+            "positive_prompt": prompt,
+            "openpose": input_openpose_url
+        }
+    })
+}
+
 export async function checkStatus(run_id: string){
     return await client.getRun(run_id)
 }
@@ -35,4 +45,4 @@ export async function getUploadUrl(type: string, file_size: number){
     } catch (error) {
         console.log(error)
     }
-}
+}