loki

newinfra/nix/modules/garage/README.md

@@ -16,9 +16,14 @@
 ## buckets
 
 - `caddy-store`: Store for Caddy webservers
-    - key `caddy`
+    - key `caddy` RW
 - `docker-registry`
+    - key `docker-registry` RW
+- `loki`
+    - key `loki` RW
 
 ## keys
 
 - `caddy`: `GK25e33d4ba20d54231e513b80`
+- `docker-registry`: `GK48011ee5b5ccbaf4233c0e40`
+- `loki`: `GK84ffae2a0728abff0f96667b`

newinfra/nix/modules/prometheus/default.nix

@@ -1,4 +1,4 @@
-{ config, ... }: {
+{ config, lib, ... }: {
   services.prometheus = {
     enable = true;
     globalConfig = { };
@@ -55,7 +55,6 @@
   };
 
   age.secrets.grafana_admin_password.file = ../../secrets/grafana_admin_password.age;
-
   systemd.services.grafana.serviceConfig.EnvironmentFile = config.age.secrets.grafana_admin_password.path;
   services.grafana = {
     enable = true;
@@ -83,8 +82,92 @@
               prometheusType = "Prometheus";
             };
           }
+          {
+            name = "loki";
+            type = "loki";
+            access = "proxy";
+            url = "http://vps3.local:3100";
+          }
         ];
       };
     };
   };
+
+  age.secrets.loki_env.file = ../../secrets/loki_env.age;
+  systemd.services.loki.serviceConfig.EnvironmentFile = config.age.secrets.loki_env.path;
+  services.loki = {
+    enable = true;
+    extraFlags = [ "-config.expand-env=true" ];
+    configuration = {
+      auth_enabled = false;
+      server = {
+        http_listen_port = 3100;
+      };
+      common = {
+        ring = {
+          instance_addr = "127.0.0.1";
+          kvstore.store = "inmemory";
+        };
+        replication_factor = 1;
+        path_prefix = "/var/lib/loki";
+      };
+      schema_config = {
+        configs = [
+          {
+            from = "2020-05-15";
+            store = "tsdb";
+            object_store = "s3";
+            schema = "v13";
+            index = {
+              prefix = "index_";
+              period = "24h";
+            };
+          }
+        ];
+      };
+      storage_config = {
+        tsdb_shipper = {
+          active_index_directory = "/var/lib/loki/index";
+          cache_location = "/var/lib/loki/cache";
+        };
+        aws = {
+          s3 = "s3://\${ACCESS_KEY}:\${SECRET_KEY}@http://127.0.0.1:3900/loki";
+          insecure = true;
+        };
+      };
+    };
+  };
+  system.activationScripts.makeLokiDir = lib.stringAfter [ "var" ] ''
+    mkdir -p /var/lib/loki/{index,cache}
+    chown ${config.services.loki.user}:${config.services.loki.group} -R /var/lib/loki
+  '';
+
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server = {
+        disable = true;
+      };
+      clients = [
+        {
+          url = "http://localhost:3100/loki/api/v1/push";
+        }
+      ];
+      scrape_configs = [
+        {
+          job_name = "journal";
+          journal = {
+            max_age = "12h";
+            labels = {
+              job = "systemd-journal";
+            };
+          };
+          relabel_configs = [{
+            source_labels = [ "__journal__systemd_unit" ];
+            target_label = "unit";
+          }];
+        }
+      ];
+    };
+  };
 }

newinfra/nix/secrets/caddy_s3_key_secret.age

@@ -1,13 +1,11 @@
 age-encryption.org/v1
--> ssh-ed25519 qM6TYg suW24NAhcPEuIWyA1lFQSrtkVIoVUQV4qRppIwoH7n4
+-> ssh-ed25519 qM6TYg CLDRFpO2DZRai0abyFUHTP0WWOBtLFS7rLOq5h5QtUs
-XzB5wU21Od/y+nFQAFVesSSPhTPlRHJNTStOJVCVKNI
+5gFHSYcctBGWkbe8LikjpTam/BHbilhbtMcWDBi9Oik
--> ssh-ed25519 XzACZQ v7FU9k53H5FZQZ2fiYElpDBcXT6+b9KNVTZ4g+2VjgI
+-> ssh-ed25519 XzACZQ kx9bB9qiKbd/SLSaDjI1qODeLyBYfUrb12qC8adCvWs
-UUulUURY2fEui2ycv6r9PsVd5sZ662Kin2ZFfdJY9AY
+UpjT6xLfv7L1DnZnVcj72KIeClbryQ1efxgHeXTjngM
--> ssh-ed25519 51bcvA 3CnO+G1LnAwYshp5DcnwfKmuFezJy0qADOzhcH6huWg
+-> ssh-ed25519 51bcvA 8nm/Z6VJacqmezgeYa1CsShZnclZgK0dfMBCgdD/unc
-B9CG7W4V9Z/oRvd54vOXsTopWiA+s3aXqypVREt+Njc
+6H2w2snEEhMvn4a4uXJdC4SfnvgQ/4B3qL7kpZ93Veg
--> ssh-ed25519 vT7ExA nmuJNZLqv2n7pTQ2f2VgmonBh63O6RSm41vqxSrCTTQ
+-> ssh-ed25519 vT7ExA Du6mW/IczVv0+SNLDT+6ghumvoNIL7wW+lKFuZ8SLTQ
-lJXeHl4mKwYoaU2lI8lgGGNkBkU9ZRpqRzxm6UrPZ0U
+aqah8fBN8JgOoi6WsrspCqqKjk4Znnl4WZQhlt/AUQ4
---- lsYfgJSFOhYj0aR8B8t83su6POtmOdkPQJw39ku/bn0
+--- ljnxwBBY5gkLMQCwJvVfS8UJJ3VH4GVxh5G6CYcJvtA
-I6óÈqöST»bcƒ7'QT¢Í“¾4¨<34>²Ã’Tt¾gÇxÀœqÐbcpŽ?l®oíC­8eÛ÷øo]ôJ*µDÐx®þ.OVßwvºãOš›
+lm)
túÐ3Á€GߎŸ¤åNÑ<4E>è=[·ç<C2B7>×?Õ­¾jóž(Ñ,{Û0œRS!“4Îé#òe¯k&ëU6E¦Ó€$¨w»
ÝvУ˜t×J6I,sÝ6Óe·nžƒ‡,ò—a÷êiÆs¤Ã”mJÔ9»–Øv~<7E>ñ<EFBFBD>Š0äøĬOl4KÔƒdÔ<18>SâŽ]ßp(d“|r
-ȹðŒ3§‘_bm 9ML}׃îÉ"„ovu"’B[Ö#›w{I%]<5D>\9™:
-m­ÏKnƒç§:/ÉP£ZÀ´I 

newinfra/nix/secrets/docker_registry_password.age

@@ -1,5 +1,5 @@
 age-encryption.org/v1
--> ssh-ed25519 qM6TYg p6LIIYNxXoumgg777+rrmMoUBuudQrDb5R6p8EW/cC0
+-> ssh-ed25519 qM6TYg NvguOs7htIflYp6bh6oiiH7Cp2l/0Mf4mcf/4b8ReQg
-bSwO9YTpnCxexbHiZANFykICHucznaA54C4hSdRnUo8
+BngCQfbilctBfNKjE+TkEhE3Bk2pkIlc1UYdAFISP/g
---- 3NVOMR4M/OUerHG/m+8srOq3JVt5D+ctjMvvkzQv47E
+--- 3hA+KfCqIAvwuL+mr4PFW9hVlpsc+t0uwG8I8Uc8JXY
-´Ez<¸¦<C2B8>IuWÿX¤<qM­Ün›Ëñˆ‡0"Oø„•¾<E280A2>ÂLÙ)1ÐƦþÝÏVá½h‘
+JŸÄ@¡zI­´9ì5/Š`…Ÿ…VfWߤæßp#™0;DÔ 
Ëدw'¬r²ÞgËÁ+n

newinfra/nix/secrets/grafana_admin_password.age

@@ -1,5 +1,6 @@
 age-encryption.org/v1
--> ssh-ed25519 XzACZQ 3tsILL/SbH8Q9HSLdHXp7bCp6qFpPvM+i9oY1ig7w2s
+-> ssh-ed25519 XzACZQ SosFhSCAHF2iDSk+H05bziuG9qOxe+/wTjQxut+KggA
-wXY15JDZ1OJ/EOcRNcptCeJL6hJm3b9Qv979+GNTmmQ
+4/f30HxHreEh28+oQwhZCP9zvg/8Wr5IVLciCWJjSmo
---- Q+TStd6iOYR9XlxmSclv//8J+PZr2M7KwK/+Wrs65zY
+--- 7p+ykQtDZWxlMpzcdjG8AMgBeo/zbrWet5A9uV5KuCo
-*<2A>·;<3B>x6ß=”<>%ä[´°Zg™·ì=:Tîs-2Ögî“
º#≮‹‚”ë½ò· ûÙha³Fû…IDr.¯fÇ¥;û‘ÕªgíØ·~DŸãˆÈ¢udÁèÕMÉf»ýdÁ˜zÞå
+‚Ò”$Åê"¨íë?0¹k—¹£¥<C2A3>ŽÏì6
'!’O4Á™¼C‰á\	¸E	#Rÿ>Mé+³+‡]êWZ	ntmñʻëdý
+¹¶KbŸU9Lâ“l“á;É}u=ÔÓ?®7¼ÛÀNy<4E>

newinfra/nix/secrets/loki_env.age

@@ -0,0 +1,6 @@
+age-encryption.org/v1
+-> ssh-ed25519 XzACZQ duC8HdUu3AuNHooD0lOyoQthZ2g7agHxE+o39iHljAk
+nOySC3inXaD1MjbosV1NcxJhXYKmU3gJu5M4CtdFwK0
+--- QkN5D8JfVCTCyBlWIou4mmV58gHZ5qgS1CY4APxm2wU
+³€'9<>NPp˜Ìí@5<>x<EFBFBD>@E“>X8˜“džæ}þ<>‰MqåGL1~¡_ áÍrxy–”Ë÷Ò0)%(DT@å[¾eÆËw2Ξ©ó…yʈBôõ¬‰}<1
+Í‹Y1Z+ÏI§y$\X1e)–µe?yáKlÑG¾\
´N£[<5B>aÁ¼ÿ
ÇYqŒôKà<1F>œ¼>ž×

newinfra/nix/secrets/secrets.nix

@@ -17,6 +17,7 @@ in
   "registry_htpasswd.age".publicKeys = [ vps1 ];
   "registry_s3_key_secret.age".publicKeys = [ vps1 ];
   "grafana_admin_password.age".publicKeys = [ vps3 ];
+  "loki_env.age".publicKeys = [ vps3 ];
   "wg_private_dns1.age".publicKeys = [ dns1 ];
   "wg_private_dns2.age".publicKeys = [ dns2 ];
   "wg_private_vps1.age".publicKeys = [ vps1 ];

newinfra/nix/secrets/wg_private_vps4.age

@@ -1,5 +1,5 @@
 age-encryption.org/v1
--> ssh-ed25519 51bcvA ji2zWkOp9u2bor9xScXWckGZN3733piHLN/gd+quiW0
+-> ssh-ed25519 51bcvA rPz/FYX2fQZl6qKVGi4lysbaEfcUlZLqgz5dTkiGEmc
-uzciBDLzZiizL3fFbn3vjiIoHGJWdFlHff3vjSWHs7g
+XFG3Mio/jSyD11sWTASw820p78mohiZ8e5vrP6ZQJO4
---- fE0bz9m5izwJX90w3RjhmzNaCPuKjhpM5M0qngI9c/A
+--- 97H29fZ0yb9XByMaOEM7RcRfsEOYwjC5C7kZERehCEU
-ð·ß/žéË3^é¥'%‹(<28>Ö¡!æk›eîG`ò<>ébÚ<62>깯ÅJ´ù×0£L.»™–д­Ê<C2AD>îp¯ ŽeŸs,<2C>1ÚÈ·øÖ
+VÙ”ïà^G~À×,s‘àô-Ù\uÒƒU(Èî2pÙ®r,½1¢¥†ÝÈ*dÎpÉ)?g/byày¢nÏMcŸ}dÌĤ3Ãv*WS`»	e¨