add victoria logs, use both

nix/modules/caddy-base/default.nix

@@ -29,7 +29,9 @@ in
     package = caddy;
     logFormat = ''
       output stdout
-      format json
+      format json {
+        message_key "_msg"
+      }
     '';
     globalConfig = ''
       email tls@noratrieb.dev

nix/modules/default/default.nix

@@ -175,6 +175,7 @@ in
       };
       clients = [
         { url = "http://loki.internal:3100/loki/api/v1/push"; }
+        { url = "http://loki.internal:9428/insert/loki/api/v1/push"; }
       ];
       scrape_configs = [
         {
@@ -207,6 +208,10 @@ in
             };
           }];
           relabel_configs = [
+            {
+              source_labels = [ "msg" ];
+              target_label = "_msg";
+            }
             {
               source_labels = [ "__journal__systemd_unit" ];
               target_label = "unit";

nix/modules/prometheus/default.nix

@@ -117,6 +117,10 @@
       };
     };
 
+    declarativePlugins = [
+      pkgs.grafanaPlugins.victoriametrics-logs-datasource
+    ];
+
     provision = {
       enable = true;
       datasources.settings = {
@@ -144,6 +148,12 @@
             access = "proxy";
             url = "http://pyroscope.internal:4040";
           }
+          {
+            name = "victorialogs";
+            type = "victoriametrics-logs-datasource";
+            access = "proxy";
+            url = "http://loki.internal:9428";
+          }
         ];
       };
     };
@@ -267,4 +277,8 @@
     createHome = true;
   };
   users.groups.pyroscope = { };
+
+  services.victorialogs = {
+    enable = true;
+  };
 }