vps/nix/modules/prometheus/default.nix

{ config, lib, networkingConfig, ... }: {
  services.prometheus = {
    enable = true;
    globalConfig = { };
    scrapeConfigs =
      let hostsWithTag = tag: map (entry: entry.name) (builtins.filter (entry: builtins.elem tag entry.value.tags) (lib.attrsToList networkingConfig)); in
      [
        {
          job_name = "prometheus";
          static_configs = [
            { targets = [ "localhost:9090" ]; }
          ];
        }
        {
          job_name = "node";
          static_configs = [{ targets = map (name: "${name}.local:9100") (builtins.attrNames networkingConfig); }];
        }
        {
          job_name = "cadvisor";
          static_configs = [{ targets = map (name: "${name}.local:8080") (builtins.attrNames networkingConfig); }];

        }
        {
          job_name = "systemd";
          static_configs = [{ targets = map (name: "${name}.local:9558") (builtins.attrNames networkingConfig); }];
        }
        {
          job_name = "caddy";
          static_configs = [{ targets = map (name: "${name}.local:9010") (hostsWithTag "apps"); }];
        }
        {
          job_name = "docker-registry";
          static_configs = [
            { targets = [ "vps1.local:9011" ]; }
          ];
        }
        {
          job_name = "garage";
          static_configs = [{ targets = map (name: "${name}.local:3903") (hostsWithTag "apps"); }];
        }
        {
          job_name = "knot";
          static_configs = [{ targets = map (name: "${name}.local:9433") (hostsWithTag "dns"); }];
        }
        {
          job_name = "pretense";
          static_configs = [{ targets = map (name: "${name}.local:9150") (builtins.attrNames networkingConfig); }];
        }
      ];
  };

  age.secrets.grafana_admin_password.file = ../../secrets/grafana_admin_password.age;
  systemd.services.grafana.serviceConfig.EnvironmentFile = config.age.secrets.grafana_admin_password.path;
  services.grafana = {
    enable = true;
    settings = {
      security = {
        admin_user = "admin";
      };
      server = {
        root_url = "https://grafana.noratrieb.dev";
      };
    };

    provision = {
      enable = true;
      datasources.settings = {
        apiVersion = 1;
        datasources = [
          {
            name = "Prometheus";
            type = "prometheus";
            access = "proxy";
            url = "http://vps3.local:9090";
            jsonData = {
              httpMethod = "POST";
              prometheusType = "Prometheus";
            };
          }
          {
            name = "loki";
            type = "loki";
            access = "proxy";
            url = "http://vps3.local:3100";
          }
        ];
      };
    };
  };

  networking.firewall.interfaces.wg0.allowedTCPPorts = [ config.services.loki.configuration.server.http_listen_port ];
  age.secrets.loki_env.file = ../../secrets/loki_env.age;
  systemd.services.loki.serviceConfig.EnvironmentFile = config.age.secrets.loki_env.path;
  services.loki = {
    enable = true;
    extraFlags = [ "-config.expand-env=true" /*"-print-config-stderr"*/ ];
    configuration = {
      auth_enabled = false;
      server = {
        http_listen_port = 3100;
      };
      common = {
        ring = {
          instance_addr = "127.0.0.1";
          kvstore.store = "inmemory";
        };
        replication_factor = 1;
        path_prefix = "/var/lib/loki";
      };
      schema_config = {
        configs = [
          {
            from = "2020-05-15";
            store = "tsdb";
            object_store = "s3";
            schema = "v13";
            index = {
              prefix = "index_";
              period = "24h";
            };
          }
        ];
      };
      storage_config = {
        tsdb_shipper = {
          active_index_directory = "/var/lib/loki/index";
          cache_location = "/var/lib/loki/cache";
        };
        aws = {
          access_key_id = "\${ACCESS_KEY}";
          secret_access_key = "\${SECRET_KEY}";
          endpoint = "127.0.0.1:3900";
          s3forcepathstyle = true;
          region = "garage";
          insecure = true;
          s3 = "s3://\${ACCESS_KEY}:\${SECRET_KEY}@127.0.0.1:3900/loki";
        };
      };
    };
  };
  system.activationScripts.makeLokiDir = lib.stringAfter [ "var" ] ''
    mkdir -p /var/lib/loki/{index,cache}
    chown ${config.services.loki.user}:${config.services.loki.group} -R /var/lib/loki
  '';
}