vllm

container.go

@@ -19,14 +19,11 @@ type Container struct {
 	IP     string `json:"ip"`
 	Image  string `json:"image"`
 	ID     string `json:"id"`
+	Engine string `json:"engine"`
 	Status string `json:"status"`
 	State  string `json:"state"`
 }
 
-type Models struct {
-	Models []Model `json:"models"`
-}
-
 func (c *Containers) Add(ctr Container) {
 	c.Containers = append(c.Containers, ctr)
 }
@@ -53,8 +50,17 @@ func get_ollama_containers(host Host, all bool) Containers {
 	}
 
 	for _, ctr := range containers {
-		if strings.Contains(ctr.Names[0], "ollama") { //&& ctr.Image == "ollama/ollama" {
-			c := Container{ID: ctr.ID, Name: ctr.Names[0], Image: ctr.Image, State: ctr.State, Status: ctr.Status}
+		if strings.Contains(ctr.Names[0], "ollama") {
+			c := Container{ID: ctr.ID, Name: ctr.Names[0], Image: ctr.Image, State: ctr.State, Status: ctr.Status, Engine: "ollama"}
+			if ctr.State == "running" {
+				// fmt.Println(ctr.Ports[0])
+				c.Port = int(ctr.Ports[0].PublicPort)
+			}
+			c.IP = host.IP
+			ctr_list.Add(c)
+			// ctr_list = append(ctr_list, c)
+		} else if strings.Contains(ctr.Names[0], "vllm") {
+			c := Container{ID: ctr.ID, Name: ctr.Names[0], Image: ctr.Image, State: ctr.State, Status: ctr.Status, Engine: "vllm"}
 			if ctr.State == "running" {
 				// fmt.Println(ctr.Ports[0])
 				c.Port = int(ctr.Ports[0].PublicPort)

host.go

@@ -19,6 +19,10 @@ type Host struct {
 	Containers []Container `json:"containers"`
 }
 
+type Models struct {
+	Models []Model `json:"models"`
+}
+
 type Model struct {
 	Container   string  `json:"container"`
 	Name        string  `json:"name"`
@@ -28,9 +32,37 @@ type Model struct {
 	Details     Details `json:"details"`
 	Ip          string  `json:"ip"`
 	Port        int     `json:"port"`
+	Engine      string  `json:"engine"`
 	State       string  `json:"state"`
 }
 
+type Vllm struct {
+	Object string `json:"object"`
+	Data   []struct {
+		ID          string      `json:"id"`
+		Object      string      `json:"object"`
+		Created     int         `json:"created"`
+		OwnedBy     string      `json:"owned_by"`
+		Root        string      `json:"root"`
+		Parent      interface{} `json:"parent"`
+		MaxModelLen int         `json:"max_model_len"`
+		Permission  []struct {
+			ID                 string      `json:"id"`
+			Object             string      `json:"object"`
+			Created            int         `json:"created"`
+			AllowCreateEngine  bool        `json:"allow_create_engine"`
+			AllowSampling      bool        `json:"allow_sampling"`
+			AllowLogprobs      bool        `json:"allow_logprobs"`
+			AllowSearchIndices bool        `json:"allow_search_indices"`
+			AllowView          bool        `json:"allow_view"`
+			AllowFineTuning    bool        `json:"allow_fine_tuning"`
+			Organization       string      `json:"organization"`
+			Group              interface{} `json:"group"`
+			IsBlocking         bool        `json:"is_blocking"`
+		} `json:"permission"`
+	} `json:"data"`
+}
+
 type Details struct {
 	Parent_model       string   `json:"parent_model"`
 	Format             string   `json:"format"`
@@ -51,37 +83,77 @@ func get_ollama_tags(hosts Hosts) (Models, error) {
 
 	retval := []Model{}
 
-	for _, ollama := range ctr.Containers {
-		if ollama.State != "running" {
-			retval = append(retval, Model{Ip: ollama.IP, Port: ollama.Port, State: "stopped"})
-		} else {
-			url := fmt.Sprintf("http://%s:%d/api/tags", ollama.IP, ollama.Port)
-			resp, err := http.Get(url)
-			if err != nil {
-				return Models{}, fmt.Errorf("failed to get %s: %v", url, err)
-			}
-			defer resp.Body.Close()
+	for _, engine := range ctr.Containers {
+		if engine.Engine == "ollama" {
+			if engine.State != "running" {
+				retval = append(retval, Model{Ip: engine.IP, Port: engine.Port, State: "stopped", Engine: "ollama"})
+			} else {
+				url := fmt.Sprintf("http://%s:%d/api/tags", engine.IP, engine.Port)
+				resp, err := http.Get(url)
+				if err != nil {
+					return Models{}, fmt.Errorf("failed to get %s: %v", url, err)
+				}
+				defer resp.Body.Close()
 
-			body, err := io.ReadAll(resp.Body)
-			if err != nil {
-				return Models{}, fmt.Errorf("failed to read body: %v", err)
-			}
+				body, err := io.ReadAll(resp.Body)
+				if err != nil {
+					return Models{}, fmt.Errorf("failed to read body: %v", err)
+				}
 
-			mods := Models{}
-			err = json.Unmarshal(body, &mods)
-			if err != nil {
-				return Models{}, fmt.Errorf("failed to unmarshal JSON: %v", err)
-			}
+				mods := Models{}
+				err = json.Unmarshal(body, &mods)
+				if err != nil {
+					return Models{}, fmt.Errorf("failed to unmarshal JSON: %v", err)
+				}
 
-			for n := range mods.Models {
-				mods.Models[n].Ip = ollama.IP
-				mods.Models[n].Port = ollama.Port
-				mods.Models[n].State = "running"
-				mods.Models[n].Container = ollama.Name
-				fmt.Printf("mod: %s - %s:%d\n", mods.Models[n].Name, mods.Models[n].Ip, mods.Models[n].Port)
-			}
+				for n := range mods.Models {
+					mods.Models[n].Ip = engine.IP
+					mods.Models[n].Port = engine.Port
+					mods.Models[n].State = "running"
+					mods.Models[n].Container = engine.Name
+					mods.Models[n].Engine = "ollama"
+					fmt.Printf("mod: %s - %s:%d\n", mods.Models[n].Name, mods.Models[n].Ip, mods.Models[n].Port)
+				}
 
-			retval = append(retval, mods.Models...)
+				retval = append(retval, mods.Models...)
+			}
+		} else if engine.Engine == "vllm" {
+			if engine.State != "running" {
+				retval = append(retval, Model{Ip: engine.IP, Port: engine.Port, State: "stopped", Engine: "vllm"})
+			} else {
+				url := fmt.Sprintf("http://%s:%d/v1/models", engine.IP, engine.Port)
+				resp, err := http.Get(url)
+				if err != nil {
+					return Models{}, fmt.Errorf("failed to get %s: %v", url, err)
+				}
+				defer resp.Body.Close()
+
+				body, err := io.ReadAll(resp.Body)
+				if err != nil {
+					return Models{}, fmt.Errorf("failed to read body: %v", err)
+				}
+
+				vllm := Vllm{}
+				err = json.Unmarshal(body, &vllm)
+				if err != nil {
+					return Models{}, fmt.Errorf("failed to unmarshal JSON: %v", err)
+				}
+
+				mods := Models{}
+				mods.Models = make([]Model, len(vllm.Data))
+
+				for n := range vllm.Data {
+					mods.Models[n].Name = vllm.Data[n].ID
+					mods.Models[n].Ip = engine.IP
+					mods.Models[n].Port = engine.Port
+					mods.Models[n].State = "running"
+					mods.Models[n].Container = engine.Name
+					mods.Models[n].Engine = "vllm"
+					fmt.Printf("mod: %s - %s:%d\n", mods.Models[n].Name, mods.Models[n].Ip, mods.Models[n].Port)
+				}
+
+				retval = append(retval, mods.Models...)
+			}
 		}
 	}
 

hosts.cfg

@@ -9,3 +9,4 @@
 
 192.168.55.10:2375,192.168.55.15:2375
 192.168.55.13:2375
+192.168.55.14:2375

static/_app/immutable/entry/start.iwPToE8J.js

@@ -1 +0,0 @@
-import{a as t}from"../chunks/entry.34uOwJQj.js";export{t as start};

static/_app/immutable/entry/start.iwPToE8J.js.br

@@ -1,2 +0,0 @@
-‹!€import{a as t}from"../chunks/entry.34uOwJQj.js";export{t as start};
-

static/_app/immutable/nodes/1.DgcYxlIK.js

@@ -1 +0,0 @@
-import{s as S,n as _,d as x}from"../chunks/scheduler.D9xsQs6S.js";import{S as j,i as q,e as d,l as f,s as y,c as g,m as h,n as v,d as l,a as C,j as m,f as $,o as E}from"../chunks/index.B2LhTP1n.js";import{s as H}from"../chunks/entry.34uOwJQj.js";const P=()=>{const s=H;return{page:{subscribe:s.page.subscribe},navigating:{subscribe:s.navigating.subscribe},updated:s.updated}},k={subscribe(s){return P().page.subscribe(s)}};function w(s){var b;let t,r=s[0].status+"",o,n,i,c=((b=s[0].error)==null?void 0:b.message)+"",u;return{c(){t=d("h1"),o=f(r),n=y(),i=d("p"),u=f(c)},l(e){t=g(e,"H1",{});var a=h(t);o=v(a,r),a.forEach(l),n=C(e),i=g(e,"P",{});var p=h(i);u=v(p,c),p.forEach(l)},m(e,a){m(e,t,a),$(t,o),m(e,n,a),m(e,i,a),$(i,u)},p(e,[a]){var p;a&1&&r!==(r=e[0].status+"")&&E(o,r),a&1&&c!==(c=((p=e[0].error)==null?void 0:p.message)+"")&&E(u,c)},i:_,o:_,d(e){e&&(l(t),l(n),l(i))}}}function z(s,t,r){let o;return x(s,k,n=>r(0,o=n)),[o]}let F=class extends j{constructor(t){super(),q(this,t,z,w,S,{})}};export{F as component};

static/_app/immutable/nodes/2.-Hdt9aq9.js

@@ -1 +0,0 @@
-import{s as X,n as J}from"../chunks/scheduler.D9xsQs6S.js";import{S as Z,i as $,e as v,s as S,c as b,g as K,a as k,m as D,d as m,b as w,j as P,f as d,p as x,l as z,n as C,o as A}from"../chunks/index.B2LhTP1n.js";function U(l){return(l==null?void 0:l.length)!==void 0?l:Array.from(l)}async function tt(){return{containers:await fetch("/api/tags",{mode:"cors",headers:{"Access-Control-Allow-Origin":"*"}}).then(l=>l.json())}}const it=Object.freeze(Object.defineProperty({__proto__:null,load:tt},Symbol.toStringTag,{value:"Module"}));function V(l,t,o){const n=l.slice();return n[1]=t[o],n}function et(l){let t,o=l[1].ip+"",n;return{c(){t=v("td"),n=z(o)},l(e){t=b(e,"TD",{});var i=D(t);n=C(i,o),i.forEach(m)},m(e,i){P(e,t,i),d(t,n)},p(e,i){i&1&&o!==(o=e[1].ip+"")&&A(n,o)},d(e){e&&m(t)}}}function lt(l){let t,o=l[1].ip+"",n,e,i=l[1].port+"",p;return{c(){t=v("td"),n=z(o),e=z(":"),p=z(i)},l(h){t=b(h,"TD",{});var r=D(t);n=C(r,o),e=C(r,":"),p=C(r,i),r.forEach(m)},m(h,r){P(h,t,r),d(t,n),d(t,e),d(t,p)},p(h,r){r&1&&o!==(o=h[1].ip+"")&&A(n,o),r&1&&i!==(i=h[1].port+"")&&A(p,i)},d(h){h&&m(t)}}}function W(l){let t,o,n=l[1].name+"",e,i,p,h,r=l[1].container+"",g,f,a,u=l[1].details.parameter_size+"",s,E,j,H=l[1].details.quantization_level+"",B,y,O,q=l[1].state+"",L,G;function I(_,c){return _[1].state=="running"?lt:et}let M=I(l),T=M(l);return{c(){t=v("tr"),o=v("td"),e=z(n),i=S(),T.c(),p=S(),h=v("td"),g=z(r),f=S(),a=v("td"),s=z(u),E=S(),j=v("td"),B=z(H),y=S(),O=v("td"),L=z(q),G=S()},l(_){t=b(_,"TR",{});var c=D(t);o=b(c,"TD",{});var N=D(o);e=C(N,n),N.forEach(m),i=k(c),T.l(c),p=k(c),h=b(c,"TD",{});var Q=D(h);g=C(Q,r),Q.forEach(m),f=k(c),a=b(c,"TD",{});var R=D(a);s=C(R,u),R.forEach(m),E=k(c),j=b(c,"TD",{});var Y=D(j);B=C(Y,H),Y.forEach(m),y=k(c),O=b(c,"TD",{});var F=D(O);L=C(F,q),F.forEach(m),G=k(c),c.forEach(m)},m(_,c){P(_,t,c),d(t,o),d(o,e),d(t,i),T.m(t,null),d(t,p),d(t,h),d(h,g),d(t,f),d(t,a),d(a,s),d(t,E),d(t,j),d(j,B),d(t,y),d(t,O),d(O,L),d(t,G)},p(_,c){c&1&&n!==(n=_[1].name+"")&&A(e,n),M===(M=I(_))&&T?T.p(_,c):(T.d(1),T=M(_),T&&(T.c(),T.m(t,p))),c&1&&r!==(r=_[1].container+"")&&A(g,r),c&1&&u!==(u=_[1].details.parameter_size+"")&&A(s,u),c&1&&H!==(H=_[1].details.quantization_level+"")&&A(B,H),c&1&&q!==(q=_[1].state+"")&&A(L,q)},d(_){_&&m(t),T.d()}}}function at(l){let t,o="GPT Backend",n,e,i,p="<tr><th>Name</th> <th>IP:Port</th> <th>Container</th> <th>Size</th> <th>Quantization</th> <th>Status</th></tr>",h,r,g=U(l[0].containers.models),f=[];for(let a=0;a<g.length;a+=1)f[a]=W(V(l,g,a));return{c(){t=v("h1"),t.textContent=o,n=S(),e=v("table"),i=v("thead"),i.innerHTML=p,h=S(),r=v("tbody");for(let a=0;a<f.length;a+=1)f[a].c();this.h()},l(a){t=b(a,"H1",{class:!0,"data-svelte-h":!0}),K(t)!=="svelte-1vaxk2u"&&(t.textContent=o),n=k(a),e=b(a,"TABLE",{class:!0});var u=D(e);i=b(u,"THEAD",{class:!0,"data-svelte-h":!0}),K(i)!=="svelte-uxk378"&&(i.innerHTML=p),h=k(u),r=b(u,"TBODY",{class:!0});var s=D(r);for(let E=0;E<f.length;E+=1)f[E].l(s);s.forEach(m),u.forEach(m),this.h()},h(){w(t,"class","h1 text-center"),w(i,"class","table-primary"),w(r,"class","table-success"),w(e,"class","table table-hover")},m(a,u){P(a,t,u),P(a,n,u),P(a,e,u),d(e,i),d(e,h),d(e,r);for(let s=0;s<f.length;s+=1)f[s]&&f[s].m(r,null)},p(a,[u]){if(u&1){g=U(a[0].containers.models);let s;for(s=0;s<g.length;s+=1){const E=V(a,g,s);f[s]?f[s].p(E,u):(f[s]=W(E),f[s].c(),f[s].m(r,null))}for(;s<f.length;s+=1)f[s].d(1);f.length=g.length}},i:J,o:J,d(a){a&&(m(t),m(n),m(e)),x(f,a)}}}function nt(l,t,o){let{data:n}=t;return l.$$set=e=>{"data"in e&&o(0,n=e.data)},[n]}class rt extends Z{constructor(t){super(),$(this,t,nt,at,X,{data:0})}}export{rt as component,it as universal};

static/_app/version.json

@@ -1 +1 @@
-{"version":"1718274879681"}
+{"version":"1732805292774"}

static/index.html

@@ -5,25 +5,25 @@
 		<link rel="icon" href="/favicon.png" />
 		<meta name="viewport" content="width=device-width, initial-scale=1" />
 		
-		<link rel="modulepreload" href="/_app/immutable/entry/start.iwPToE8J.js">
-		<link rel="modulepreload" href="/_app/immutable/chunks/entry.34uOwJQj.js">
+		<link rel="modulepreload" href="/_app/immutable/entry/start.cMTxM74J.js">
+		<link rel="modulepreload" href="/_app/immutable/chunks/entry.CkeGJzPd.js">
 		<link rel="modulepreload" href="/_app/immutable/chunks/scheduler.D9xsQs6S.js">
-		<link rel="modulepreload" href="/_app/immutable/entry/app.BUn2_CVn.js">
+		<link rel="modulepreload" href="/_app/immutable/entry/app.DkzyH-q8.js">
 		<link rel="modulepreload" href="/_app/immutable/chunks/index.B2LhTP1n.js">
 	</head>
 	<body data-sveltekit-preload-data="hover">
 		<div style="display: contents">
 			<script>
 				{
-					__sveltekit_1gocnci = {
+					__sveltekit_1b4sbwy = {
 						base: ""
 					};
 
 					const element = document.currentScript.parentElement;
 
 					Promise.all([
-						import("/_app/immutable/entry/start.iwPToE8J.js"),
-						import("/_app/immutable/entry/app.BUn2_CVn.js")
+						import("/_app/immutable/entry/start.cMTxM74J.js"),
+						import("/_app/immutable/entry/app.DkzyH-q8.js")
 					]).then(([kit, app]) => {
 						kit.start(app, element);
 					});