diff --git a/lib.nix b/lib.nix index 9ef15a0c..58fa6038 100644 --- a/lib.nix +++ b/lib.nix @@ -35,6 +35,19 @@ then addr else trimAddress6 addrReplaced; + bindToAddress = { + localhost ? null, + localhost4 ? coalesce [localhost "127.0.0.1"], + localhost6 ? coalesce [localhost "::1"], + }: listen: + if listen == "localhost" + then coalesce [localhost listen] + else if listen == "0.0.0.0" + then localhost4 + else if getAddress6 listen == "::" + then localhost6 + else listen; + parseUrl = url: let parts' = Regex.match ''^([^:]+)://(\[[0-9a-fA-F:]+]|[^/:\[]+)(|:[0-9]+)(|/.*)$'' url; parts = parts'.value; @@ -60,6 +73,10 @@ if Str.hasInfix ":" addr && ! Str.hasPrefix "[" addr then "[${addr}]" else addr; + getAddress6 = addr: + if Str.hasInfix ":" addr + then Str.removePrefix "[" (Str.removeSuffix "]" addr) + else addr; coalesce = values: Opt.default null (List.find (v: v != null) values); mapListToAttrs = f: l: listToAttrs (map f l); @@ -116,7 +133,9 @@ in { mkWinPath mkBaseDn mkAddress6 + getAddress6 trimAddress6 + bindToAddress mapListToAttrs coalesce mkAlmostOptionDefault diff --git a/modules/nixos/nginx/upstream.nix b/modules/nixos/nginx/upstream.nix index da5dd3c8..4d0365c6 100644 --- a/modules/nixos/nginx/upstream.nix +++ b/modules/nixos/nginx/upstream.nix @@ -60,6 +60,7 @@ let (mkIf (cfg.id != null) (mkAlmostOptionDefault (access.systemForServiceId cfg.id).name)) (mkOptionDefault (mapNullable (serviceName: (access.systemForService serviceName).name) cfg.name)) ]; + network = mkIf (port.listen == "tail") (mkAlmostOptionDefault "tail"); }; conf = { enable = lib.warnIf (!port.enable) "${cfg.system}.exports.services.${cfg.name}.ports.${cfg.port} isn't enabled" ( diff --git a/modules/nixos/nginx/vouch.nix b/modules/nixos/nginx/vouch.nix index 14640f70..15af376d 100644 --- a/modules/nixos/nginx/vouch.nix +++ b/modules/nixos/nginx/vouch.nix @@ -5,7 +5,7 @@ gensokyo-zone, ... }: let - inherit (gensokyo-zone.lib) mkAlmostOptionDefault; + inherit (gensokyo-zone.lib) mkAlmostOptionDefault bindToAddress; inherit (lib.options) mkOption mkEnableOption; inherit (lib.modules) mkIf mkMerge mkBefore mkAfter mkOptionDefault mkDefault; inherit (lib.attrsets) mapAttrsToList; @@ -296,10 +296,7 @@ in { upstreams' = let localVouch = let inherit (vouch-proxy.settings.vouch) listen port; - host = - if listen == "0.0.0.0" || listen == "[::]" - then "localhost" - else listen; + host = bindToAddress {localhost = "localhost";} listen; in { # TODO: accessService.exportedId = "login"; enable = mkAlmostOptionDefault vouch-proxy.enable; diff --git a/modules/system/exports/ollama.nix b/modules/system/exports/ollama.nix new file mode 100644 index 00000000..eebef875 --- /dev/null +++ b/modules/system/exports/ollama.nix @@ -0,0 +1,30 @@ +{ + lib, + gensokyo-zone, + ... +}: let + inherit (gensokyo-zone.lib) mkAlmostOptionDefault; + inherit (lib.modules) mkIf; +in { + config.exports.services.ollama = {config, ...}: { + displayName = mkAlmostOptionDefault "Ollama"; + id = mkAlmostOptionDefault "ollama"; + nixos = { + serviceAttr = "ollama"; + assertions = mkIf config.enable [ + (nixosConfig: { + assertion = config.ports.default.port == nixosConfig.services.ollama.port; + message = "port mismatch"; + }) + ]; + }; + defaults.port.listen = mkAlmostOptionDefault "lan"; + ports = { + default = { + port = mkAlmostOptionDefault 11434; + protocol = "http"; + status.enable = mkAlmostOptionDefault true; + }; + }; + }; +} diff --git a/modules/system/exports/services.nix b/modules/system/exports/services.nix index 9dde5fd3..f5fadf85 100644 --- a/modules/system/exports/services.nix +++ b/modules/system/exports/services.nix @@ -38,7 +38,7 @@ else config.name; }; listen = mkOption { - type = enum ["wan" "lan" "int" "localhost"]; + type = enum ["wan" "lan" "int" "tail" "localhost"]; }; protocol = mkOption { type = nullOr (enum ["http" "https"]); diff --git a/nixos/access/nextjs-ollama.nix b/nixos/access/nextjs-ollama.nix new file mode 100644 index 00000000..b3c6d977 --- /dev/null +++ b/nixos/access/nextjs-ollama.nix @@ -0,0 +1,103 @@ +{ + config, + gensokyo-zone, + lib, + ... +}: let + inherit (gensokyo-zone.lib) bindToAddress; + inherit (lib.modules) mkIf mkBefore mkDefault; + inherit (lib.strings) escapeRegex; + inherit (config.services) tailscale; + cfg = config.services.nextjs-ollama-llm-ui; + upstreamName = "ollama'nextjs"; +in { + services.nextjs-ollama-llm-ui = { + #ollamaUrl = mkDefault "https://${virtualHost.serverName}/ollama"; + }; + services.nginx = { + upstreams'.${upstreamName}.servers = { + local = { + enable = mkDefault cfg.enable; + addr = mkDefault (bindToAddress {} cfg.hostname); + port = mkIf cfg.enable (mkDefault cfg.port); + }; + }; + virtualHosts = let + name.shortServer = "lm"; + copyFromVhost = mkDefault "llama"; + vouch = { + enable = true; + requireAuth = false; + }; + subFilterLocation = { virtualHost, ... }: mkIf (virtualHost.locations ? "/ollama/") { + proxy.headers.set.Accept-Encoding = ""; + extraConfig = '' + sub_filter_once off; + sub_filter_types application/javascript; + sub_filter '${cfg.ollamaUrl}' '/ollama'; + ''; + }; + proxyLocation = { + imports = [ subFilterLocation ]; + proxy = { + enable = true; + upstream = mkDefault upstreamName; + }; + }; + locations = { + "~ ^/llama$" = { + return = mkDefault "302 /llama/"; + }; + "/llama/" = {virtualHost, ...}: { + imports = [ proxyLocation ]; + vouch.requireAuth = mkIf virtualHost.vouch.enable true; + proxy.path = "/"; + }; + "/_next/" = {virtualHost, ...}: { + imports = [ proxyLocation ]; + vouch.requireAuth = mkIf virtualHost.vouch.enable true; + }; + "/_next/static/" = _: { + imports = [ proxyLocation ]; + }; + "~ '^/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'" = { + return = mkDefault "302 /llama$request_uri"; + }; + "/" = {virtualHost, ...}: { + extraConfig = mkBefore '' + if ($http_referer ~ '^https?://${escapeRegex virtualHost.serverName}/llama/') { + return 302 /llama$request_uri; + } + ''; + return = mkDefault "404"; + }; + }; + in { + llama = { + inherit name locations vouch; + ssl.force = true; + }; + llama'local = { + inherit locations; + name = { + inherit (name) shortServer; + includeTailscale = false; + }; + ssl.cert = { + inherit copyFromVhost; + }; + local.enable = mkDefault true; + }; + llama'tail = { + inherit locations; + enable = mkDefault tailscale.enable; + name = { + inherit (name) shortServer; + qualifier = mkDefault "tail"; + }; + ssl.cert.copyFromVhost = "llama'local"; + local.enable = mkDefault true; + }; + }; + }; +} diff --git a/nixos/access/ollama.nix b/nixos/access/ollama.nix new file mode 100644 index 00000000..cb7ef520 --- /dev/null +++ b/nixos/access/ollama.nix @@ -0,0 +1,81 @@ +{ + config, + gensokyo-zone, + lib, + ... +}: let + inherit (gensokyo-zone.lib) bindToAddress; + inherit (lib.modules) mkIf mkDefault; + inherit (config.services) tailscale; + cfg = config.services.ollama; + requestTimeout = "${toString (60 * 60)}s"; + upstreamName = "ollama'access"; +in { + services.nginx = { + upstreams'.${upstreamName}.servers = { + local = { + enable = mkDefault cfg.enable; + addr = mkDefault (bindToAddress {} cfg.host); + port = mkIf cfg.enable (mkDefault cfg.port); + }; + service = {upstream, ...}: { + enable = mkIf upstream.servers.local.enable (mkDefault false); + accessService.name = "ollama"; + #settings.fail_timeout = mkDefault requestTimeout; + }; + }; + virtualHosts = let + name.shortServer = "lm"; + copyFromVhost = mkDefault "llama"; + locations = { + "/ollama/" = {virtualHost, ...}: { + vouch.requireAuth = mkIf virtualHost.vouch.enable (mkDefault true); + proxy = { + enable = true; + upstream = upstreamName; + path = "/"; + }; + extraConfig = '' + proxy_buffering off; + proxy_read_timeout ${requestTimeout}; + ''; + headers.set.Access-Control-Allow-Origin = "https://${virtualHost.serverName}/llama/"; + }; + "/".return = mkDefault "404"; + }; + vouch = { + enable = true; + requireAuth = false; + }; + in { + llama = { + inherit name locations vouch; + ssl.force = true; + }; + llama'local = { + inherit locations vouch; + name = { + inherit (name) shortServer; + includeTailscale = false; + }; + ssl = { + force = true; + cert = { + inherit copyFromVhost; + }; + }; + local.enable = mkDefault true; + }; + llama'tail = { + inherit locations; + enable = mkDefault tailscale.enable; + name = { + inherit (name) shortServer; + qualifier = mkDefault "tail"; + }; + ssl.cert.copyFromVhost = "llama'local"; + local.enable = mkDefault true; + }; + }; + }; +} diff --git a/nixos/home-assistant.nix b/nixos/home-assistant.nix index f4f56126..4550aabd 100644 --- a/nixos/home-assistant.nix +++ b/nixos/home-assistant.nix @@ -180,10 +180,14 @@ in { "cast" "nfandroidtv" "octoprint" + "ollama" "plex" "shopping_list" "tile" "wake_on_lan" + "wyoming" + "whisper" + "piper" "withings" "wled" ]; diff --git a/nixos/ollama/nextjs.nix b/nixos/ollama/nextjs.nix new file mode 100644 index 00000000..1cef16d1 --- /dev/null +++ b/nixos/ollama/nextjs.nix @@ -0,0 +1,18 @@ +{ + pkgs, + config, + gensokyo-zone, + access, + lib, + ... +}: let + inherit (gensokyo-zone.lib) mkAlmostOptionDefault; + inherit (lib.modules) mkDefault; +in { + services.nextjs-ollama-llm-ui = { + enable = mkDefault true; + package = mkAlmostOptionDefault pkgs.nextjs-ollama-llm-ui-develop; + ollamaUrl = mkAlmostOptionDefault (access.proxyUrlFor {serviceName = "ollama";}); + port = mkAlmostOptionDefault 3001; + }; +} diff --git a/overlays/default.nix b/overlays/default.nix index dd683257..e1f9f7d7 100644 --- a/overlays/default.nix +++ b/overlays/default.nix @@ -5,6 +5,7 @@ in rec { barcodebuddy builders krb5 + llm minecraft nfs nginx @@ -14,6 +15,7 @@ in rec { ]; barcodebuddy = import ./barcodebuddy.nix; krb5 = import ./krb5.nix; + llm = import ./llm.nix; minecraft = import ./minecraft.nix; nfs = import ./nfs.nix; nginx = import ./nginx.nix; diff --git a/overlays/llm.nix b/overlays/llm.nix new file mode 100644 index 00000000..a392bd6c --- /dev/null +++ b/overlays/llm.nix @@ -0,0 +1,42 @@ +final: prev: let + inherit (final) lib; +in { + ollama-mmap = prev.ollama.overrideAttrs (old: { + postPatch = + '' + substituteInPlace api/types.go \ + --replace 'UseMMap: nil,' 'UseMMap: &[]bool{true}[0],' + '' + + old.postPatch or ""; + doCheck = false; + }); + ollama-cuda = final.ollama.override { + acceleration = "cuda"; + }; + ollama-rocm = final.ollama.override { + acceleration = "rocm"; + }; + + nextjs-ollama-llm-ui-develop = prev.nextjs-ollama-llm-ui.overrideAttrs (old: rec { + version = "2024-08-27"; + name = "${old.pname}-${version}"; + + patches = let + packageRoot = final.path + "/pkgs/by-name/ne/nextjs-ollama-llm-ui"; + in [ + #(packageRoot + "/0001-update-nextjs.patch") + (packageRoot + "/0002-use-local-google-fonts.patch") + #(packageRoot + "/0003-add-standalone-output.patch") + ]; + + src = old.src.override { + rev = "7c8eb67c3eb4f18eaa9bde8007147520e3261867"; + hash = "sha256-Ym5RL+HbOmOM6CLYFf0JMsM+jMcFyCUAm1bD/CXeE+I="; + }; + npmDeps = final.fetchNpmDeps { + name = "${name}-npm-deps"; + hash = "sha256-8VRBUNUDwSQYhRJjqaKP/RwUgFKKoiQUPjGDFw37Wd4="; + inherit src patches; + }; + }); +} diff --git a/systems/hakurei/nixos.nix b/systems/hakurei/nixos.nix index fc64d25a..3dcb816e 100644 --- a/systems/hakurei/nixos.nix +++ b/systems/hakurei/nixos.nix @@ -44,6 +44,8 @@ in { nixos.access.kitchencam nixos.access.moonraker nixos.access.mpd + nixos.access.ollama + nixos.access.nextjs-ollama nixos.access.openwebrx nixos.access.deluge nixos.access.home-assistant @@ -53,6 +55,7 @@ in { nixos.access.proxmox nixos.access.plex nixos.access.invidious + nixos.ollama.nextjs nixos.wake-chen nixos.samba nixos.syncplay @@ -276,6 +279,15 @@ in { virtualHosts.mpd'local.allServerNames ]; }; + lm = { + inherit (nginx) group; + domain = virtualHosts.llama.serverName; + extraDomainNames = mkMerge [ + virtualHosts.llama.otherServerNames + virtualHosts.llama'local.allServerNames + (mkIf virtualHosts.llama'tail.enable virtualHosts.llama'tail.allServerNames) + ]; + }; webrx = { inherit (nginx) group; domain = virtualHosts.openwebrx.serverName; @@ -409,6 +421,7 @@ in { moonraker.ssl.cert.enable = true; openwebrx.ssl.cert.enable = true; mpd.ssl.cert.enable = true; + llama.ssl.cert.enable = true; deluge.ssl.cert.enable = true; invidious = { ssl.cert.enable = true; diff --git a/systems/shanghai/default.nix b/systems/shanghai/default.nix index 689d6974..585afe87 100644 --- a/systems/shanghai/default.nix +++ b/systems/shanghai/default.nix @@ -41,5 +41,10 @@ in { ports.public.port = 32022; }; prometheus-exporters-node.enable = true; + ollama = { + enable = true; + defaults.port.listen = "tail"; + ports.default.status.enable = false; + }; }; } diff --git a/tf/cloudflare_records.tf b/tf/cloudflare_records.tf index 41aff54a..732b73d0 100644 --- a/tf/cloudflare_records.tf +++ b/tf/cloudflare_records.tf @@ -30,6 +30,7 @@ module "hakurei_system_records" { "kitchen", "print", "radio", + "lm", "webrx", "deluge", "home", @@ -50,6 +51,7 @@ module "hakurei_system_records" { "kitchen", "print", "radio", + "lm", "webrx", "syncplay", "yt",