feat(access): a llama

lib.nix

@@ -35,6 +35,19 @@
       then addr
       else trimAddress6 addrReplaced;
 
+  bindToAddress = {
+    localhost ? null,
+    localhost4 ? coalesce [localhost "127.0.0.1"],
+    localhost6 ? coalesce [localhost "::1"],
+  }: listen:
+    if listen == "localhost"
+    then coalesce [localhost listen]
+    else if listen == "0.0.0.0"
+    then localhost4
+    else if getAddress6 listen == "::"
+    then localhost6
+    else listen;
+
   parseUrl = url: let
     parts' = Regex.match ''^([^:]+)://(\[[0-9a-fA-F:]+]|[^/:\[]+)(|:[0-9]+)(|/.*)$'' url;
     parts = parts'.value;
@@ -60,6 +73,10 @@
     if Str.hasInfix ":" addr && ! Str.hasPrefix "[" addr
     then "[${addr}]"
     else addr;
+  getAddress6 = addr:
+    if Str.hasInfix ":" addr
+    then Str.removePrefix "[" (Str.removeSuffix "]" addr)
+    else addr;
 
   coalesce = values: Opt.default null (List.find (v: v != null) values);
   mapListToAttrs = f: l: listToAttrs (map f l);
@@ -116,7 +133,9 @@ in {
       mkWinPath
       mkBaseDn
       mkAddress6
+      getAddress6
       trimAddress6
+      bindToAddress
       mapListToAttrs
       coalesce
       mkAlmostOptionDefault

modules/nixos/nginx/upstream.nix

@@ -60,6 +60,7 @@ let
           (mkIf (cfg.id != null) (mkAlmostOptionDefault (access.systemForServiceId cfg.id).name))
           (mkOptionDefault (mapNullable (serviceName: (access.systemForService serviceName).name) cfg.name))
         ];
+        network = mkIf (port.listen == "tail") (mkAlmostOptionDefault "tail");
       };
       conf = {
         enable = lib.warnIf (!port.enable) "${cfg.system}.exports.services.${cfg.name}.ports.${cfg.port} isn't enabled" (

modules/nixos/nginx/vouch.nix

@@ -5,7 +5,7 @@
   gensokyo-zone,
   ...
 }: let
-  inherit (gensokyo-zone.lib) mkAlmostOptionDefault;
+  inherit (gensokyo-zone.lib) mkAlmostOptionDefault bindToAddress;
   inherit (lib.options) mkOption mkEnableOption;
   inherit (lib.modules) mkIf mkMerge mkBefore mkAfter mkOptionDefault mkDefault;
   inherit (lib.attrsets) mapAttrsToList;
@@ -296,10 +296,7 @@ in {
     upstreams' = let
       localVouch = let
         inherit (vouch-proxy.settings.vouch) listen port;
-        host =
+        host = bindToAddress {localhost = "localhost";} listen;
-          if listen == "0.0.0.0" || listen == "[::]"
-          then "localhost"
-          else listen;
       in {
         # TODO: accessService.exportedId = "login";
         enable = mkAlmostOptionDefault vouch-proxy.enable;

modules/system/exports/ollama.nix

@@ -0,0 +1,30 @@
+{
+  lib,
+  gensokyo-zone,
+  ...
+}: let
+  inherit (gensokyo-zone.lib) mkAlmostOptionDefault;
+  inherit (lib.modules) mkIf;
+in {
+  config.exports.services.ollama = {config, ...}: {
+    displayName = mkAlmostOptionDefault "Ollama";
+    id = mkAlmostOptionDefault "ollama";
+    nixos = {
+      serviceAttr = "ollama";
+      assertions = mkIf config.enable [
+        (nixosConfig: {
+          assertion = config.ports.default.port == nixosConfig.services.ollama.port;
+          message = "port mismatch";
+        })
+      ];
+    };
+    defaults.port.listen = mkAlmostOptionDefault "lan";
+    ports = {
+      default = {
+        port = mkAlmostOptionDefault 11434;
+        protocol = "http";
+        status.enable = mkAlmostOptionDefault true;
+      };
+    };
+  };
+}

modules/system/exports/services.nix

@@ -38,7 +38,7 @@
           else config.name;
       };
       listen = mkOption {
-        type = enum ["wan" "lan" "int" "localhost"];
+        type = enum ["wan" "lan" "int" "tail" "localhost"];
       };
       protocol = mkOption {
         type = nullOr (enum ["http" "https"]);

nixos/access/nextjs-ollama.nix

@@ -0,0 +1,103 @@
+{
+  config,
+  gensokyo-zone,
+  lib,
+  ...
+}: let
+  inherit (gensokyo-zone.lib) bindToAddress;
+  inherit (lib.modules) mkIf mkBefore mkDefault;
+  inherit (lib.strings) escapeRegex;
+  inherit (config.services) tailscale;
+  cfg = config.services.nextjs-ollama-llm-ui;
+  upstreamName = "ollama'nextjs";
+in {
+  services.nextjs-ollama-llm-ui = {
+    #ollamaUrl = mkDefault "https://${virtualHost.serverName}/ollama";
+  };
+  services.nginx = {
+    upstreams'.${upstreamName}.servers = {
+      local = {
+        enable = mkDefault cfg.enable;
+        addr = mkDefault (bindToAddress {} cfg.hostname);
+        port = mkIf cfg.enable (mkDefault cfg.port);
+      };
+    };
+    virtualHosts = let
+      name.shortServer = "lm";
+      copyFromVhost = mkDefault "llama";
+      vouch = {
+        enable = true;
+        requireAuth = false;
+      };
+      subFilterLocation = { virtualHost, ... }: mkIf (virtualHost.locations ? "/ollama/") {
+        proxy.headers.set.Accept-Encoding = "";
+        extraConfig = ''
+          sub_filter_once off;
+          sub_filter_types application/javascript;
+          sub_filter '${cfg.ollamaUrl}' '/ollama';
+        '';
+      };
+      proxyLocation = {
+        imports = [ subFilterLocation ];
+        proxy = {
+          enable = true;
+          upstream = mkDefault upstreamName;
+        };
+      };
+      locations = {
+        "~ ^/llama$" = {
+          return = mkDefault "302 /llama/";
+        };
+        "/llama/" = {virtualHost, ...}: {
+          imports = [ proxyLocation ];
+          vouch.requireAuth = mkIf virtualHost.vouch.enable true;
+          proxy.path = "/";
+        };
+        "/_next/" = {virtualHost, ...}: {
+          imports = [ proxyLocation ];
+          vouch.requireAuth = mkIf virtualHost.vouch.enable true;
+        };
+        "/_next/static/" = _: {
+          imports = [ proxyLocation ];
+        };
+        "~ '^/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'" = {
+          return = mkDefault "302 /llama$request_uri";
+        };
+        "/" = {virtualHost, ...}: {
+          extraConfig = mkBefore ''
+            if ($http_referer ~ '^https?://${escapeRegex virtualHost.serverName}/llama/') {
+              return 302 /llama$request_uri;
+            }
+          '';
+          return = mkDefault "404";
+        };
+      };
+    in {
+      llama = {
+        inherit name locations vouch;
+        ssl.force = true;
+      };
+      llama'local = {
+        inherit locations;
+        name = {
+          inherit (name) shortServer;
+          includeTailscale = false;
+        };
+        ssl.cert = {
+          inherit copyFromVhost;
+        };
+        local.enable = mkDefault true;
+      };
+      llama'tail = {
+        inherit locations;
+        enable = mkDefault tailscale.enable;
+        name = {
+          inherit (name) shortServer;
+          qualifier = mkDefault "tail";
+        };
+        ssl.cert.copyFromVhost = "llama'local";
+        local.enable = mkDefault true;
+      };
+    };
+  };
+}

nixos/access/ollama.nix

@@ -0,0 +1,81 @@
+{
+  config,
+  gensokyo-zone,
+  lib,
+  ...
+}: let
+  inherit (gensokyo-zone.lib) bindToAddress;
+  inherit (lib.modules) mkIf mkDefault;
+  inherit (config.services) tailscale;
+  cfg = config.services.ollama;
+  requestTimeout = "${toString (60 * 60)}s";
+  upstreamName = "ollama'access";
+in {
+  services.nginx = {
+    upstreams'.${upstreamName}.servers = {
+      local = {
+        enable = mkDefault cfg.enable;
+        addr = mkDefault (bindToAddress {} cfg.host);
+        port = mkIf cfg.enable (mkDefault cfg.port);
+      };
+      service = {upstream, ...}: {
+        enable = mkIf upstream.servers.local.enable (mkDefault false);
+        accessService.name = "ollama";
+        #settings.fail_timeout = mkDefault requestTimeout;
+      };
+    };
+    virtualHosts = let
+      name.shortServer = "lm";
+      copyFromVhost = mkDefault "llama";
+      locations = {
+        "/ollama/" = {virtualHost, ...}: {
+          vouch.requireAuth = mkIf virtualHost.vouch.enable (mkDefault true);
+          proxy = {
+            enable = true;
+            upstream = upstreamName;
+            path = "/";
+          };
+          extraConfig = ''
+            proxy_buffering off;
+            proxy_read_timeout ${requestTimeout};
+          '';
+          headers.set.Access-Control-Allow-Origin = "https://${virtualHost.serverName}/llama/";
+        };
+        "/".return = mkDefault "404";
+      };
+      vouch = {
+        enable = true;
+        requireAuth = false;
+      };
+    in {
+      llama = {
+        inherit name locations vouch;
+        ssl.force = true;
+      };
+      llama'local = {
+        inherit locations vouch;
+        name = {
+          inherit (name) shortServer;
+          includeTailscale = false;
+        };
+        ssl = {
+          force = true;
+          cert = {
+            inherit copyFromVhost;
+          };
+        };
+        local.enable = mkDefault true;
+      };
+      llama'tail = {
+        inherit locations;
+        enable = mkDefault tailscale.enable;
+        name = {
+          inherit (name) shortServer;
+          qualifier = mkDefault "tail";
+        };
+        ssl.cert.copyFromVhost = "llama'local";
+        local.enable = mkDefault true;
+      };
+    };
+  };
+}

nixos/home-assistant.nix

@@ -180,10 +180,14 @@ in {
       "cast"
       "nfandroidtv"
       "octoprint"
+      "ollama"
       "plex"
       "shopping_list"
       "tile"
       "wake_on_lan"
+      "wyoming"
+      "whisper"
+      "piper"
       "withings"
       "wled"
     ];

nixos/ollama/nextjs.nix

@@ -0,0 +1,18 @@
+{
+  pkgs,
+  config,
+  gensokyo-zone,
+  access,
+  lib,
+  ...
+}: let
+  inherit (gensokyo-zone.lib) mkAlmostOptionDefault;
+  inherit (lib.modules) mkDefault;
+in {
+  services.nextjs-ollama-llm-ui = {
+    enable = mkDefault true;
+    package = mkAlmostOptionDefault pkgs.nextjs-ollama-llm-ui-develop;
+    ollamaUrl = mkAlmostOptionDefault (access.proxyUrlFor {serviceName = "ollama";});
+    port = mkAlmostOptionDefault 3001;
+  };
+}

overlays/default.nix

@@ -5,6 +5,7 @@ in rec {
     barcodebuddy
     builders
     krb5
+    llm
     minecraft
     nfs
     nginx
@@ -14,6 +15,7 @@ in rec {
   ];
   barcodebuddy = import ./barcodebuddy.nix;
   krb5 = import ./krb5.nix;
+  llm = import ./llm.nix;
   minecraft = import ./minecraft.nix;
   nfs = import ./nfs.nix;
   nginx = import ./nginx.nix;

overlays/llm.nix

@@ -0,0 +1,42 @@
+final: prev: let
+  inherit (final) lib;
+in {
+  ollama-mmap = prev.ollama.overrideAttrs (old: {
+    postPatch =
+      ''
+        substituteInPlace api/types.go \
+          --replace 'UseMMap:   nil,' 'UseMMap:   &[]bool{true}[0],'
+      ''
+      + old.postPatch or "";
+    doCheck = false;
+  });
+  ollama-cuda = final.ollama.override {
+    acceleration = "cuda";
+  };
+  ollama-rocm = final.ollama.override {
+    acceleration = "rocm";
+  };
+
+  nextjs-ollama-llm-ui-develop = prev.nextjs-ollama-llm-ui.overrideAttrs (old: rec {
+    version = "2024-08-27";
+    name = "${old.pname}-${version}";
+
+    patches = let
+      packageRoot = final.path + "/pkgs/by-name/ne/nextjs-ollama-llm-ui";
+    in [
+      #(packageRoot + "/0001-update-nextjs.patch")
+      (packageRoot + "/0002-use-local-google-fonts.patch")
+      #(packageRoot + "/0003-add-standalone-output.patch")
+    ];
+
+    src = old.src.override {
+      rev = "7c8eb67c3eb4f18eaa9bde8007147520e3261867";
+      hash = "sha256-Ym5RL+HbOmOM6CLYFf0JMsM+jMcFyCUAm1bD/CXeE+I=";
+    };
+    npmDeps = final.fetchNpmDeps {
+      name = "${name}-npm-deps";
+      hash = "sha256-8VRBUNUDwSQYhRJjqaKP/RwUgFKKoiQUPjGDFw37Wd4=";
+      inherit src patches;
+    };
+  });
+}

systems/hakurei/nixos.nix

@@ -44,6 +44,8 @@ in {
     nixos.access.kitchencam
     nixos.access.moonraker
     nixos.access.mpd
+    nixos.access.ollama
+    nixos.access.nextjs-ollama
     nixos.access.openwebrx
     nixos.access.deluge
     nixos.access.home-assistant
@@ -53,6 +55,7 @@ in {
     nixos.access.proxmox
     nixos.access.plex
     nixos.access.invidious
+    nixos.ollama.nextjs
     nixos.wake-chen
     nixos.samba
     nixos.syncplay
@@ -276,6 +279,15 @@ in {
         virtualHosts.mpd'local.allServerNames
       ];
     };
+    lm = {
+      inherit (nginx) group;
+      domain = virtualHosts.llama.serverName;
+      extraDomainNames = mkMerge [
+        virtualHosts.llama.otherServerNames
+        virtualHosts.llama'local.allServerNames
+        (mkIf virtualHosts.llama'tail.enable virtualHosts.llama'tail.allServerNames)
+      ];
+    };
     webrx = {
       inherit (nginx) group;
       domain = virtualHosts.openwebrx.serverName;
@@ -409,6 +421,7 @@ in {
       moonraker.ssl.cert.enable = true;
       openwebrx.ssl.cert.enable = true;
       mpd.ssl.cert.enable = true;
+      llama.ssl.cert.enable = true;
       deluge.ssl.cert.enable = true;
       invidious = {
         ssl.cert.enable = true;

systems/shanghai/default.nix

@@ -41,5 +41,10 @@ in {
       ports.public.port = 32022;
     };
     prometheus-exporters-node.enable = true;
+    ollama = {
+      enable = true;
+      defaults.port.listen = "tail";
+      ports.default.status.enable = false;
+    };
   };
 }

tf/cloudflare_records.tf

@@ -30,6 +30,7 @@ module "hakurei_system_records" {
     "kitchen",
     "print",
     "radio",
+    "lm",
     "webrx",
     "deluge",
     "home",
@@ -50,6 +51,7 @@ module "hakurei_system_records" {
     "kitchen",
     "print",
     "radio",
+    "lm",
     "webrx",
     "syncplay",
     "yt",