diff --git a/worker/runner.gen.go b/worker/runner.gen.go index b5e26950..b5ddefc7 100644 --- a/worker/runner.gen.go +++ b/worker/runner.gen.go @@ -272,9 +272,10 @@ type ImageToTextResponse struct { // LLMChoice defines model for LLMChoice. type LLMChoice struct { - Delta LLMMessage `json:"delta"` - FinishReason *string `json:"finish_reason,omitempty"` - Index int `json:"index"` + Delta *LLMMessage `json:"delta,omitempty"` + FinishReason *string `json:"finish_reason,omitempty"` + Index int `json:"index"` + Message *LLMMessage `json:"message,omitempty"` } // LLMMessage defines model for LLMMessage. @@ -3005,89 +3006,89 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xde3Pbtpb/KhjtzjiZkfxq0+545v7huGniuXbq8eO2nTajC5FHEmISYAHQtpr1d9/B", - "AUECJCjJru12e/VXZBKP8/ydA+AQ+TJIRF4IDlyrwcGXgUrmkFP8eXh2/E5KIc3vFFQiWaGZ4IMD84aA", - "eUUkqEJwBSQXKWTbg+GgkKIAqRngGLmadbtfzqHqnoNSdAamn2Y6g8HB4FTNzF+LwvyhtGR8Nri/Hw4k", - "/FYyCeng4Bcc9VPTpSa07icmnyHRg/vh4LBMmTivqOySch7QT6ZCEmp6kBlwkNS06jKFLcyP/5YwHRwM", - "/munkeFOJcCdU0gZvTo/6RBve/vkByRGeHgr0sV4BhwbXopLuNNm9h6iQgavikzQFNKKrSnLgGhBJkC0", - "pNy0nEBqWJwKmVM9OBhMGKdyMWjR19XJcJCDpinV1M46pWVm+n+5Hwzb9pKmzPykGfksJoRxOxkTvKKl", - "oEpBav7QcyAFKyBjPDQLN1eMDqO7MUtDOjpUfChnM8Zn5HuaOH0ff0dKM7HRu5NH4ZReT22bprGpJehS", - "8rFmOShN80KFNGhZQoeOc+xDmj52+nmgEqLhTm+Ti7IohNSQkhualaAOyJYCroEnsDUkW7dCpltDYqyW", - "WKLIRIgMKCevtszkW+bd1pRmCrZeb5PvLGWEKVK9ftWM93rbtSQ5UK4IFx6R29Vs1TvzezShqLWmjSe1", - "isvLRjKrvLrjGDG7X+IexzmdwaXAf7r+MStZSnkCY5XQDAI1fbv9pq2jdzwRpaQzUJWl6BoSgLAcXySZ", - "UJAtSMb4dWO8Rm+kkCIvNHk1Z7M5yEp3JKcLIiEtk2oI8ltJM6YXr325va/oJBdIZ80vL/MJSMMvcwz2", - "eLodWwtDOZsuyC3T845f9bu7lV/E1nHc8RI57nXl+B3MJCAxt3OWWDKcHB2lTJGiVHMU4S2VqcJWjDPN", - "aGbbbLfpI6vFlAlJ1QpIOCQn4vyQvDoRt6Nzyq/JYUoLjcj0ulI85SlhWpFESBvsUuNlt8Bmc42Oa5nw", - "4gV5d0fzIoMD8oX8OsioBq5HieCKKeNoi50syUeGupFK77JfBwdkb3t3SH4dcJDss9op2B1kIyr1yL3d", - "v/cFcIKMPRsOdvhZEwo5zKhmNzC2xr+CiMvGTV6p1+heJUuB3M6pNn/BXZKVKZCpFHlExMczLqSxoCkJ", - "DZL8Wu7ufpWQPZ/sjxVp5MySFqO+zMfWr8cFyBgPe20WPqKpETF1gOBjRAGyYi8gpMzJsW18BrJDDuMa", - "ZtZ6kR4+BQnImoZWaNnb3e2nJwUumDI6xo7b5FRIsL9JqUqaGdQCiphVQVQFRY6VSamJysQtSFJTYYZJ", - "yww9d7Iw8Qb4TM87/Ln25AKpjnHni3cdq1hmk/06VXQKejFO5pBcB8Izoa8tvTOQBhNNIMVuBLuhKSrN", - "csT9aRu7DCyUWWpSGDGdAlfGyIQkcyrzaZn5ZF7YUY+QmJrYKlojtQBpVyIXULmlpDwVObH41iMK0zgq", - "b6erQAq72//TA9dialORJk2jRZGxJshJcDq2mnm1a97sBYHsws3ZweZW3C+cAm1giyQAQWRfnQHEE+S1", - "w2bN+pNFzidMUGuVrAvLfwiN+6fs87qWblepdM2c7l8sBdFV6bQFit8MIyvNqaQ5KARkBYngKZp3kIfc", - "mOF97r7vwa05hv1gzjffRme1LQnjBMO5WmPSD3bw2Lxr224df6gdH+Pnn2q1loyHpxO5MK3HkzK5Bt2m", - "Ym//2zYZV25Co2JcbRqijMhpLkqujQLsmPVyy08oUGc2FJpXFcyan7mJnVXPW5ZlBuwZx1cdFZ7aZm+R", - "6IAxP7QLpmBMy9m4B5Z39zt5as0CdiY0TRswDhi26TL5ECw8qkWHBAX5JMO0ubevTXh5IoEqx3cQ4pGA", - "w3JG+gF+dfqy/+b/cfayySucJG5Z2rLevd39r2N4iC0fBIc/4tjdWR8YYWzoWBJiLmCWA9eHfKHnjM/2", - "u2FmIu4ie6AkQwMiXxMqJV2QGbsBTqgilEzEndsCqPwMcXFo+P/p559+JhaNfW7firveNXd38mOH98oS", - "/1iEp+p6zHhR6ih/4nYkQYmsRFAzjQk2bjGlFwVL0CtxsUZJIeGGiVKZHylLsDfTlV0Nm6wK/WLv7sPd", - "j+TVh3/8+I/9N9+gSV4cngaZ5KmZ+RjJ/MutevMyM16srsei1LUgl+DBscmtSxg2ErRRRVa7gnOTgJsB", - "7bYgzSdsVhphWtFbs1JDIqYauPkzLRPc9wOtQVY99ZxygziMzzLw1BBw5SgnP1jKY+DBjVFl7HcYJ0LI", - "VD2MvUIwrgn2ZJxqUHUArcdtlhSUz4D8sjvc+1SZCPau5iVwV0CibfMJ2AYSlHloHln1pSw3WCm4CiNW", - "NRc5sjzEGPUn6zrDx7v9ysvFtOKqUkTLF27nIIEATSryCTOKI69+Gv78ukG/IJHGZm3KvPwdCcvoBLII", - "YSf4vM5oAtIcNXuE8ZQlKH9qmsJMipKnVWsT73eDJhOaXPtNuuTaaZdsiGdixvQDrMV2U6TkI+MBai4y", - "k+GgedqxCONKm6gvpoZExDh8H9l0PrGzd/W8buzoxIQl8eOqqHdCH7ngfOJ92qcBxNKylT5+P3BFCvjt", - "m/+gDay1pLnZyVqVcT5458g5Z8R/j+Ylv47lPYl5gQmqUSZ6JW0OubrHwbrabuomvThAleniqD6L4dZH", - "o+t6pp4x3evOwExDbgi69+aox6onwjDWkaT2GxrCPFlaQUUk+P7s6kjkRanhmE8jZ8+n9Zl6CpoyY/7v", - "z65IYvv4x8BdoVr4qrEunnvRz7Y6oUkUP/tVAJ6/5pALuRhPJUDQAR+T783jJd200DSL9LvE59GOjLdI", - "wwfR7QCaBzR9NH+v3FczAuG2ZUBkyKqTkSPI02pLeXH1XmmWsd9RRatUbDRbNs2J0lQzpVmiHqncF9bY", - "emoYDjwex5Ul+908iZFKvtHp/GEszX2jWMrXWAivaxIxDqIEhdbStoWIxXy4vDzrKRkyr9asGbJgsaq+", - "pi76acuh6u4R35DVQ/O/aMZS5Kymvo8kB7JLaWuP5yHyd3akCBz71LYHiNFNZXpLJXpvhaJrlTcZP12K", - "vLOiNHkbOjutC3fOgjbLmG8Bi8fZ+6IkfabjJ61rHWtUObLf/sw9W4WeRdOwnnfYMO6bTkTKS5RxoalW", - "a6lBAs1GJuqiQpaBpSFLuXEfqZC277aUYsn+62rFsh9RSy/hH4Bmen7k8uVQoma4UsXTqzl2JLaJS7E8", - "yoCXuSH2h38OhoN35+c/nA+Gg+PvTt755F3YCVYxXNHh8+WRHeEKl3wPKmaMLTciq9QeYbTXCH6Wubr4", - "0bcyW/SwKgutaPnU6resNtI7R3yQYDA1XyaX/qy+kQrW6K1M6dupdiu9jnEQYfTk5PRoLlgCseiU2SLM", - "ZSo5OTk9tdW2ZrQp40zNxxKocnHD2yioj0GxFTm3rWLb0zyFuyCXwwcrcxVLsevvCaPhMi4Cx0JHBong", - "Grj2aTmqHsV2qUQWYNS5yFbjk7SN3Ewh1Y6wONnn8FsJKlKUkNO7sRbXwNvHY9/429935NK2iWe8OLNa", - "OzsJLaFJkKth2j5aoXGvkWA8iG5aaAk0D/pheWlYIkLz6B6ChrwwXlZKaJ2Ofuv7W9MocgipRTEOt0tG", - "e15nUZB/RiVq+hXt2i+/29nKmpZaKaGVODvos5IGwFrWjT7xIBVXbnTvr+PtIF0FJxIMngXOUz2KViOs", - "s3xzNrPSTKz5j0tlCVjBFjrClTXelsyThj9vyKFdGOXV9I7Vtl6WAq83aQR48iIDXDY1jtxAkHu5zIPt", - "Nlaku90QXNYVV3iRnrjO7e8Y3UdzwwwjPLVmCqXniScmP3YDeAhbncWeUUmtBXcRXIpsXMpsxc711fkJ", - "xnBVTrBanvEZuWGUXEqWXON5l9AiEVm1j53isUZVspGxm6puY6TFqF2cQgokzo/pR5YsciWj1gs3xkIf", - "QHRRTjKm5oZm27efdJd2umyU8pRkYhaQ986O0UPdmkcBH2kO7VyXaEFkybtyMy/sj89isk0+Cs0SIBrr", - "dudMEaYIpzmkxE3uqnJc/bPdUhV6DpJIUWpQQ9yuZ5qkAhThQtsCQzMTJdFzCluwA3c00fbZK/WapFAA", - "TxURPOSEGWvOgeuqlpGnJMfqrAkWwEzZrJR0kgFqwvT8tzWDfxMqZ6U7X19r/VPbdi3tL/edQ6+qsh0b", - "gwbpnd9Fvnyp/CXiWZUpNcYXlnIqzbjl2NhepV5R6pmwBysm9hoRV8MEc9pHfWblHA/iU1+IUibgz8p4", - "IvJw1noMooOyj4v6eXTy9hIqoCQUiQ9ScQxaA60etKhYDi7dNcbDIQ/RzBW2LZ/uGTHMB14tHI4tg6y1", - "EetpbZpo8edZ9Usb9bIk5pSqa/UgW7Z9XYFRjwH7x/ztZbKkt0NScq/So6lDUeSV7fq6hj4sXAk/cwkP", - "8cOypZU7Fp3xUARRvSdC9u2AoDy2lA0TKR4K2+ZIN9Z5hFMGUGYHXvk1a0WYcs0rqX5q0b5Uv7jhEjnM", - "zM0Lp0yDN5TZWk/vG0w6EaVuleNhv67CuZredqf5cQ7aVc7aCW+pItOMzmaQEqrIx4vvfwyO2c0w6x8d", - "G02YN7Y6wS9zrmdcq1wx6tdmcOPUtlimYSGh3CQINElAKfuNbH08s4YTW9dVlhQUm69PVFefHq/OT2Kq", - "RPSVIq8+peulMtTYS/Pc5tIwE2H06Tfs8BxcrbNlZ4/M19/NtAfg960T8dhC+nk3DYeOx09h72XAYN5X", - "X870Lbz+Pl/GPuXnGZ3vTpd8nrH51HTzqenf91PTN//RX5qSCzALdQ0ES70Lu0mDpb+4j7H1v1vGNFR9", - "UcNk0RQEb6r7/rTvSTr4veb3JJXBtEJsGEJ74+xFAZDM+wJtwIUPWYckN3iiCqDXIEkKZmUvldFxZsA/", - "WxC4KyQo1JsJE5SjqlPTB5K5q1I0Roe2ah6n2LJgOkHP6Syl3V9Gdm5qs4TVAFW6Zf6y48f16A3yjJ+9", - "rkPJsmjRJGXLQ4T9wAL3N5ZN1ZuvhfYSmELEYFbWHGUiCc57KF/8MB0c/PKlw+GXjk1/uvdjeNKqW2kO", - "9uztTK0jwKgM8UHTFGkml+bpqtTV8GGnqlp6rrVGndPDd+FW77vZT4JXJeruA1rTNlgrPLAEor1GcIfr", - "logVJREVqb7Mlu/1IEInpWR6cWFIsXx+uLw8ewtUgqwv+UJYt4/qQeZaF4P7ezzZj5V7HlY3AST15U2y", - "5OTwuN738zf6TtgNFAZLDo/Jeck5TmRwzY61u727vWsEIgrgtGCDg8FX23vbu0ZbVM+R7B28E2ikxcg5", - "cSFULJrXFyd591zZb3Oq1ZYoKms4Ts1Son2pkLQHs29FumhVE9ioT6XeMWF35O67smpeZQSxG4zuQxWb", - "GI8PrEKR7f3d3RYVntR3PlclG+uRECwQce5W4C5xsT8tM9I0Gw6+fkISmvrLyPxvaUrcsTjOu/cy815x", - "Wuq5kOx3SHHiva9eZuKKWfKOa5MGXwpBTqi0xRhf7715Ke6bhBWRymK5IWF//0lJ6NTCdolpmpC6XvbN", - "S9nfMdcgOc3IBcgbkKQpKnYwijHXB9BfPt1/Gg5UmedULtwFeeRSEJca0Jky2O1CiUHvu5FNsahajDjN", - "YSRuQEqWIvIH6DAc7Myr8sYdh8IzQBGEIObXpg6eEUFiNbDrAsm9Lyc3kC0CDjmtK1yXsurqPZ+dVzvR", - "H+PSjWHYxLrOfvbs6+fkyyssfRxXlkTkBpdWJijXXzjGo/JhUWQL95ljcJOMskf7hRQmyfIWa50w3br6", - "55njdDDbCwfqsNR1E6n7I/UmQj00Qtn7Ii4FqT8afmCIYqFj+CCwRmaOG1YWB1Yn5uHNUC/j8H9GYh6r", - "+954/V88P99Az6Oh55HJMQs81Aeem/pSuCjyvI9dhfagpMNdHfQyGGRne2EQCjeTNvCzSTqewfPrK7ge", - "5/rOMYaDnYzdwCiseFy1/IguPLxqZlu7519tqkvJISXAU7z9RkUhol18txQmHq+jnsLVF0aJ3krDDWBs", - "AOPpAMOYmQWLP4IaWdszLXJk+RqpAp41lljPQElG+aw0EFYf5XdR4OT0uRy/+VjspZ3d+xxq498b/35C", - "/0ZvebA/Z7l14aoUfUSrW9lG+/0eXV3gVhU+49dflC/J+CMXvj1z1t+Z8YXdPCwp3zj6xtGfztGd9znj", - "JvuP8HvVdZDhYMdE6DWOHt63KpJx7e8VIMeTeq/S65nCereWbHPKsHH7v4nbYxXdHzhk0J77Bc5u6/HW", - "2uoLu/j/6539z8rct8FuE1A3lX+Up14JZvBfwfUgha3xe1aoCMoIXxgrwv+YcIMVG6x4eqyoXehxYFF1", - "R7QovYuYozBRXQZbrwTIZOH+pwv8JFIr0tx3H3X75jrZZ14duIk22cHG4/8mHu9dxfxAVy99Z1BIgMLp", - "Wnfhu3rjo0yUKTkSeV5yphfkPdVwSxeD6gNgrHJWBzs7qQSaj2b27XZWdd9OTHcsq+8Z/0JjVtE3bD2Q", - "wnY7tGA7E9B0p+b3/tP9/wUAAP//rAXUV3h5AAA=", + "H4sIAAAAAAAC/+xdeW/ctrb/KsS8B9gBZry1aR8M3D8cJ02Ma6eGl5sWbTCXI53RMJZIlaRsT/P83R94", + "qIWUqFlc2+3rnb8ylric9XcOySPm6yASWS44cK0Gh18HKppBRvHn0fnJOymFNL9jUJFkuWaCDw7NGwLm", + "FZGgcsEVkEzEkO4MhoNcihykZoBjZCrpdr+aQdk9A6VoAqafZjqFweHgTCXmr3lu/lBaMp4MHh6GAwm/", + "FUxCPDj8BUf93HSpCa37ickXiPTgYTg4KmImLkoqu6RcePSTqZCEmh4kAQ6SmlZdprCF+fHfEqaDw8F/", + "7TYy3C0FuHsGMaPXF6cd4m1vl3yPxAAPb0Q8HyfAseGVuIJ7bWbvIcpn8DpPBY0hLtmashSIFmQCREvK", + "TcsJxIbFqZAZ1YPDwYRxKueDFn1dnQwHGWgaU03trFNapKb/14fBsG0vcczMT5qSL2JCGLeTMcFLWnKq", + "FMTmDz0DkrMcUsZ9s6jmCtFhdDdmsU9Hh4oPRZIwnpAfaFTp++QtKczERu+VPPJK6fXUtmkcmlqCLiQf", + "a5aB0jTLlU+DlgV06LjAPqTpY6efeSohGu71Drks8lxIDTG5pWkB6pBsKeAaeARbQ7J1J2S8NSTGaokl", + "ikyESIFysr1lJt8y77amNFWw9WqHvLWUEaZI+Xq7Ge/VTtWSZEC5Ilw4RO6Us5XvzO/RhKLWmjaO1Eou", + "rxrJLPPqjmOE7H6Be5xkNIErgf90/SMpWEx5BGMV0RQ8NX2/87qto3c8EoWkCajSUnQNCUBYhi+iVChI", + "5yRl/KYxXqM3kkuR5Zpsz1gyA1nqjmR0TiTERVQOQX4raMr0/JUrt/clneQS6az55UU2AWn4ZRWDPZ5u", + "x9bCUM6mc3LH9KzjV/3ubuUXsHUcd7xAjvtdOb6FRAISczdjkSWjkmNFKVMkL9QMRXhHZaywFeNMM5ra", + "Njtt+shyMaVCUrUEEo7Iqbg4Itun4m50QfkNOYpprhGZXpWKpzwmTCsSCWmDXWy87A5YMtPouJYJJ16Q", + "d/c0y1M4JF/Jr4OUauB6FAmumDKONt9No2xkqBup+D79dXBI9nf2huTXAQfJvqjdnN1DOqJSj6q3Bw+u", + "AE6RsWfDwQ4/K0Ihh4Rqdgtja/xLiLhq3GRbvUL3KlgM5G5GtfkL7qO0iIFMpcgCIj5JuJDGgqbEN0jy", + "a7G3901E9l2yP5akkXNLWoj6Ihtbvx7nIEM87LdZ+IimRsS0AgQXI3KQJXseIUVGTmzjc5AdchjXkFjr", + "RXr4FCQgaxpaoWV/b6+fnhi4YMroGDvukDMhwf4mhSpoalALKGJWCVElFFWsTApNVCruQJKaCjNMXKTo", + "uZO5iTfAEz3r8Fe1J5dIdYg7V7yrWMUim+zXqaJT0PNxNIPoxhOeCX1t6Z2DNJhoAil2I9gNTVFpliHu", + "T9vYZWChSGOTwojpFLgyRiYkmVGZTYvUJfPSjnqMxNTEltEaqQWIuxK5hNItJeWxyIjFtx5RmMZBeVe6", + "8qSwt/M/PXAtpjYVadI0mucpa4KchErHVjPbe+bNvhfILqs5O9jcivt5pUAb2AIJgBfZl2cA4QR55bBZ", + "s/5kkfMJE9RaJavC8h9C4/4p+7yupdtlKl0xp/sXi0F0VTptgeJ3w8BKcyppBgoBWUEkeIzm7eUht2Z4", + "l7sfenBrhmHfm/P198FZbUvCOMFwrlaY9IMdPDTvyrZbxx9qx8f4+adarSVj/XQiE6b1eFJEN6DbVOwf", + "fN8m47qa0KgYV5uGKCNymomCa6MAO2a93HITCtSZDYXmVQmz5mdmYmfZ846lqQF7xvFVR4VnttkbJNpj", + "zA3tgikY0yIZ98Dy3kEnT61ZwM6ExnEDxh7DNl0mH7yFR7nokKAgm6SYNvf2tQkvjyRQVfHthXgk4KhI", + "SD/AL09fDl7/P85eNnlFJYk7Fresd3/v4NsQHmLLteDwE47dnXXNCGNDx4IQcwlJBlwf8bmeMZ4cdMPM", + "RNwH9kBJigZEviVUSjonCbsFTqgilEzEfbUFUPoZ4uLQ8P/Tzz/9TCwau9y+Efe9a+7u5CcV3itL/GMR", + "nqqbMeN5oYP8ibuRBCXSAkHNNCbYuMWUnucsQq/ExRoluYRbJgplfsQswt5Ml3Y1bLIq9Iv9+w/3n8j2", + "h398+sfB6+/QJC+PzrxM8szMfIJk/uVWvVmRGi9WN2NR6FqQC/DgxOTWBQwbCdqoIstdwZlJwM2AdluQ", + "ZhOWFEaYVvTWrNSQiKkGbv6Miwj3/UBrkGVPPaPcIA7jSQqOGjyuKsrJj5byEHhwY1Qp+x3GkRAyVuux", + "lwvGNcGejFMNqg6g9bjNkoLyBMgve8P9z6WJYO9yXgL3OUTaNp+AbSBBmYfmkVVfzDKDlYIrP2KVc5Fj", + "y0OIUXeyrjN8vD8ovVxMS65KRbR84W4GEgjQqCSfMKM4sv3T8OdXDfp5iTQ2a1Pm5O9IWEonkAYIO8Xn", + "dUbjkVZRs08Yj1mE8qemKSRSFDwuW5t4v+c1mdDoxm3SJddOu2BDPBUJ02tYi+2mSMFHxgPUTKQmw0Hz", + "tGMRxpU2UV9MDYmIcfg+sOl8amfv6nnV2NGJCQvix3Ve74Q+csH5xPu0TwOIhWUrfvx+4JIU8PvX/0Eb", + "WCtJc7OTtSzjXHvnqHLOgP8ezwp+E8p7IvMCE1SjTPRK2hxydY+Ddbnd1E16cYAy08VRXRb9rY9G1/VM", + "PWNWrzsDMw2ZIejBmaMeq54Iw1hHktptaAhzZGkFFZDg+/PrY5HlhYYTPg2cPZ/VZ+oxaMqM+b8/vyaR", + "7eMeA3eFauGrxrpw7kW/2OqEJlH84lYBOP6aQSbkfDyVAF4HfEx+MI8XdNNC0zTQ7wqfBzsy3iINHwS3", + "A2jm0fTR/L10X80IhNuWHpE+q5WMKoIcrbaUF1bvtWYp+x1VtEzFRrNF05woTTVTmkXqkcp9YY2tpobh", + "wOFxXFqy282RGCnlG5zOHcbS3DeKpXyFhfCqJhHiIEiQby1tWwhYzIerq/OekiHzasWaIQsWy+pr6qKf", + "thzK7g7xDVk9NP+LpixGzmrq+0iqQHYhbe3xHER+a0cKwLFLbXuAEN1UxndUoveWKLpSeZPx04XIm+SF", + "ydvQ2WlduHPutVnEfAtYHM7e5wXpMx03aV3pWKPMkd3259WzZeiZNw3reYcN467pBKS8QBmXmmq1khok", + "0HRkoi4qZBFYGrJUNe4jFdL23ZZSLNl/Xa1Y9gNq6SX8A9BUz46rfNmXqBmuUOH0aoYdiW1SpVgOZcCL", + "zBD74z8Hw8G7i4sfLwbDwcnb03cueZd2gmUMl3S4fDlkB7jCJd9axYyh5UZgldojjPYawc0ylxc/ulZm", + "ix6WZaElLZ9b/RbVRjrniGsJBlPzRXLpz+obqWCN3tKUvp1qt9LrEAcBRk9Pz45ngkUQik6pLcJcpJLT", + "07MzW21rRpsyztRsLIGqKm44GwX1MSi2Ihe2VWh7msdw7+Vy+CCcmNnJ1yCzbR04tiO5RiRheZ01U/oC", + "iwTXwLVL+HH5KLSlJVIP0C5EuhzMpG1UzeRTXREWJvsCfitABSoYMno/1uIGePss7Tt3r/yeXNk2C7Sg", + "Vk5lfLNpsulymLZDl9Dda1EYPII7HFoCzbx+WIvq15PQLLjhoCHLjUsWElpHqd+7ztk0CpxYapGP/b2V", + "0b7TWeTkn0GJmn55u1DM7Xa+tACmVopvJZUd9FlJg3Yt60afWEvFpRs9uIt+O0hXwZEEA36e85SPgqUL", + "q6z1KptZaibW/MeFsgQsYQsd4ToMJnGVZgwalvzxh7Uk23pZiNLOpAHgyfIUcI3VOHIDQdXLRR5s97wC", + "3e3u4aKuuBwM9MRFcX/H4KZbNcwwwFNrJl96jnhC8mO3gCe25cHtOZXUWnAXwaVIx4VMl2xzX1+cYsBX", + "xQRL6xlPyC2j5Eqy6AYPx4QWkUjLTe8Yz0DK+o6U3ZZFHiMtRu1KFpIjcW4CcGzJItcyaL1wayx0DaLz", + "YpIyNTM02779pFc5apW6Uh6TVCQeee/sGD3UrXhu8JFm0E6MiRZEFrwrN/PC/vgiJjvko9AsAqKxyHfG", + "FGGKcJpBTKrJqxKeqlja7r8KPQNJpCg0qCHu7TNNYgGKcKFtNaKZiZLgoYat7oF7Gmn7bFu9IjHkwGNF", + "BPc5YcaaM+C6LHzkMcmwlGuC1TJTlhSSTlJATZie/7Zm8G9CZVJUh/ErLZZq266l/fWhc0JWlsFjY9Ag", + "ncO+wGcypb8EPKs0pcb4/LpPpRm3HBvbK9UrCp0IewpjYq8RcTmMN6d91GdWleNBeOpLUcgI3FkZj0Tm", + "z1qPQbRXI3JZPw9O3l5veZT4InFBKoxBK6DVWiuQxeDSXZCsD3mIZlUV3OLpnhHDXODVosKxRZC1MmI9", + "rU0TLf48q35po16UxJxRdaPWsmXbt6pG6jFgtyagvaaW9G5ICu6UhTRFK4ps266vaujDKhf/mxj/xN+v", + "cVq6vdEZD0UQ1HskZN92CcpjS9kwEeMJsm2OdGNRiD+lB2V24KWfvpaEqap5KdXPLdoX6hd3ZwInn5l5", + "USnT4A1ltjDU+WCTTkShW7V72K+rcK6md91pPs1AV2W2dsI7qsg0pUkCMaGKfLz84ZN3Jm+GWf2c2WjC", + "vLGlDG5NdD3jSrWNQb82gxuntpU1DQsR5SZBoFEEStkPauuznBWc2LqusqSg2Fx9orr69Hh9cRpSJaKv", + "FFn53V0vlb7GXprnNpeGmQCjT7+7h4fmapX9PXu+vvrWpz0tf2gdn4cW0s+7wzisePzs914EDOZ9+ZlN", + "38Lr7/MZ7VN+y9H5SHXBtxyb71I336X+fb9Lff0f/VkquQSzUNdAsC48t5s0WCeM+xhb/7tlTEPVtzpM", + "5k318KYU8E/7+KSD3yt+fFIaTCvE+iG0N85e5gDRrC/Qely4kHVEMoMnKgd6A5LEYFb2Uhkdpwb80zmB", + "+1yCQr2ZMEE5qjo2fSCaVSWNxujQVs3jGFvmTEfoOZ2ldPWXkV01tVnCaoAy3TJ/2fHDenQGecZvZFeh", + "ZFG0aJKyxSHCfo2B+xuLpurN13x78UwhYDBLC5RSEXnnPZTPf5wODn/52uHwa8emPz+4MTxqFbk0B3v2", + "KqfWEWBQhvigaYo0kyvzdFnqaviwU5UtHddaoShq/V245ftu9vvhZYl69bWtaeutFdasl2ivEaqTeEvE", + "kvqJklRXZov3ehCho0IyPb80pFg+P1xdnb8BKkHWN4IhrNtH9SAzrfPBwwOWAYRqQ4/KawOi+qYnWXBy", + "dFLv+7kbfafsFnKDJUcn5KLgHCcyuGbH2tvZ29kzAhE5cJqzweHgm539nT2jLapnSPYuXiA00mJUOXEu", + "VCia17csOZdi2Q95ytWWyEtrOInNUqJ9A5G0B7NvRDxvVRPYqE+l3jVhd1RdjmXVvMwIQtcdPfgqNjEe", + "H1iFItsHe3stKhyp734p6ztWI8FbIOLcrcBd4GJ/WqSkaTYcfPuEJDTFmoH539CYVMfiOO/+y8x7zWmh", + "Z0Ky3yHGife/eZmJS2bJO65NGnwlBDml0hZjfLv/+qW4bxJWRCqL5YaEg4MnJaFTONslpmlC6uLa1y9l", + "fydcg+Q0JZcgb0GSpgK5glGMuS6A/vL54fNwoIoso3Je3aZHrgSpUgOaKIPdVSgx6H0/sikWVfMRpxmM", + "xC1IyWJEfg8dhoPdWVkLuVuhcAIoAh/E3ELWwTMiSKhgdlUgeXDlVA1kK4Z9Tuty2IWsVsWhz86rneiP", + "cVmNYdjEItB+9uzr5+TLqUJ9HFeWROQGl1YmKNefQ4aj8lGep/Pqm0jv2hllj/ZzKUyS5SzWOmG6dU/Q", + "M8dpb7YXDtR+XewmUvdH6k2EWjdC2cslrgSpvzBeM0Qx3zFcEFghM8cNK4sDyxNz/xqpl3H4PyMxDxWJ", + "b7z+L56fb6Dn0dDzyOSYeR7qAs9tfYNcEHneh+5NWyvpqO4ZehkMsrO9MAj5m0kb+NkkHc/g+fV9XY9z", + "/coxhoPdlN3CyK94XLb8CC48nGpmW7vn3oOqC8khJsBjvCpHBSGiXXy3ECYer6OewtUXRoneSsMNYGwA", + "4+kAw5iZBYs/ghpp2zMtcqTZCqkCnjUWWM9ASUp5UhgIq4/yuyhwevZcjt98LPbSzu58DrXx741/P6F/", + "o7es7c9pZl24LEUf0fIKt9FBv0eXt72Vhc/49RflCzL+wO1wz5z1d2Z8YTf3S8o3jr5x9Kdz9Mr7KuMm", + "B4/we9V1kOFg10ToFY4e3rcqknHt7xQgh5N6p9LrmcJ6t5Zsc8qwcfu/idtjFd0fOGTQjvt5zm7r8Vba", + "6vO7uP9Fnv2fzapvg6tNQN1U/lEeOyWY3v8b14MUtsbvWaHCKyN8Yazw/xfDDVZssOLpsaJ2oceBRdkd", + "0aJwbm0OwkR5c2y9EiCTefXfYuAnkVqR5nL8oNs3d88+8+qgmmiTHWw8/m/i8c69zWu6euE6g0ICFE7X", + "uji/qjc+TkURk2ORZQVnek7eUw13dD4oPwDGKmd1uLsbS6DZKLFvd9Ky+05kumNZfc/4lxqzir5h64EU", + "ttulOdudgKa7Nb8Pnx/+LwAA//86n8VApXkAAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/worker/worker.go b/worker/worker.go index 6c5a2f4e..7ec429cf 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -397,7 +397,6 @@ func (w *Worker) AudioToText(ctx context.Context, req GenAudioToTextMultipartReq func (w *Worker) LLM(ctx context.Context, req GenLLMJSONRequestBody) (interface{}, error) { isStreaming := req.Stream != nil && *req.Stream ctx, cancel := context.WithCancel(ctx) - defer cancel() c, err := w.borrowContainer(ctx, "llm", *req.Model) if err != nil { return nil, err @@ -419,8 +418,8 @@ func (w *Worker) LLM(ctx context.Context, req GenLLMJSONRequestBody) (interface{ } return w.handleStreamingResponse(ctx, c, resp, cancel) } - defer cancel() + defer cancel() resp, err := c.Client.GenLLMWithResponse(ctx, req) if err != nil { return nil, err @@ -762,25 +761,18 @@ func (w *Worker) handleNonStreamingResponse(c *RunnerContainer, resp *GenLLMResp return resp.JSON200, nil } -type LlmStreamChunk struct { - Chunk string `json:"chunk,omitempty"` - TokensUsed int `json:"tokens_used,omitempty"` - Done bool `json:"done,omitempty"` -} - -func (w *Worker) handleStreamingResponse(ctx context.Context, c *RunnerContainer, resp *http.Response, returnContainer func()) (<-chan LlmStreamChunk, error) { +func (w *Worker) handleStreamingResponse(ctx context.Context, c *RunnerContainer, resp *http.Response, returnContainer func()) (<-chan *LLMResponse, error) { if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) } - outputChan := make(chan LlmStreamChunk, 10) + outputChan := make(chan *LLMResponse, 10) go func() { defer close(outputChan) defer returnContainer() scanner := bufio.NewScanner(resp.Body) - totalTokens := 0 for scanner.Scan() { select { @@ -788,26 +780,22 @@ func (w *Worker) handleStreamingResponse(ctx context.Context, c *RunnerContainer return default: line := scanner.Text() - if strings.HasPrefix(line, "data: ") { - data := strings.TrimPrefix(line, "data: ") - if data == "[DONE]" { - outputChan <- LlmStreamChunk{Chunk: "[DONE]", Done: true, TokensUsed: totalTokens} - return - } - - var streamData LlmStreamChunk - if err := json.Unmarshal([]byte(data), &streamData); err != nil { - slog.Error("Error unmarshaling stream data", slog.String("err", err.Error())) - continue - } - - totalTokens += streamData.TokensUsed - - select { - case outputChan <- streamData: - case <-ctx.Done(): - return - } + data := strings.TrimPrefix(line, "data: ") + + if data == "[DONE]" { + break + } + + var llmRes *LLMResponse + if err := json.Unmarshal([]byte(data), llmRes); err != nil { + slog.Error("Error unmarshaling stream data", slog.String("err", err.Error())) + continue + } + + select { + case outputChan <- llmRes: + case <-ctx.Done(): + return } } }