.dev.vars

@@ -1 +0,0 @@
-AUTHENTICATION_TOKEN=foobar

.dockerignore

@@ -0,0 +1,17 @@
+# flyctl launch added from .gitignore
+# ---> Rust
+# Generated by Cargo
+# will have compiled files and executables
+**/debug
+**/target
+
+# These are backup files generated by rustfmt
+**/**/*.rs.bk
+
+# MSVC Windows builds of rustc generate these, which store debugging information
+**/*.pdb
+
+**/.direnv
+**/gs.*
+**/result
+fly.toml

.envrc

@@ -1 +1 @@
-use flake
+use flake

.forgejo/workflows/publish-posts.yml

@@ -1,28 +0,0 @@
-name: Publish posts
-on:
-  push:
-    paths:
-      - 'posts/**'
-      - '.forgejo/workflows/publish-posts.yml'
-
-jobs:
-  sync:
-    runs-on: debian-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3.5.3
-        with:
-          ref: trunk
-      - name: Install AWS CLI
-        run: |
-          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-          unzip awscliv2.zip
-          ./aws/install
-      - name: Sync with S3
-        run: |
-          aws s3 sync ./posts s3://gabrielsimmer-com/posts --endpoint-url=$AWS_ENDPOINT --size-only --delete --output text
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }}
-          AWS_DEFAULT_REGION: 'weur'
-          AWS_ENDPOINT: ${{ secrets.AWS_ENDPOINT }}

.gitea/workflows/flyctl-deploy.yml

@@ -0,0 +1,31 @@
+name: Fly Deploy
+on:
+  push:
+    branches:
+      - trunk
+jobs:
+  deploy:
+    name: Deploy app
+    runs-on: debian-latest-arm
+    steps:
+      - uses: actions/checkout@v3.6.0
+        with:
+          ref: trunk
+      - uses: https://github.com/superfly/flyctl-actions/setup-flyctl@master
+      - run: flyctl deploy --remote-only
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+  smoketest:
+    name: Smoketest
+    needs: deploy
+    runs-on: debian-latest-arm
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3.6.0
+      - name: Run k6 local test
+        uses: https://github.com/grafana/k6-action@v0.3.0
+        with:
+          filename: k6/smoketest.js
+          flags: -o experimental-prometheus-rw --tag testid=smoketest-${GITHUB_SHA}
+        env:
+          K6_PROMETHEUS_RW_SERVER_URL: ${{ secrets.PROMETHEUS_URL }}

.gitignore

@@ -1,7 +1,15 @@
-/.direnv
-result
+# ---> Rust
+# Generated by Cargo
+# will have compiled files and executables
+debug/
 target/
-node_modules/
-wrangler/
-.wrangler/
-result/
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+# MSVC Windows builds of rustc generate these, which store debugging information
+*.pdb
+
+.direnv/
+gs.*
+result

Cargo.toml

@@ -1,2 +1,34 @@
-[workspace]
-members = ["queues", "web"]
+[package]
+name = "gabrielsimmerdotcom"
+version = "0.1.0"
+edition = "2021"
+
+[profile.release]
+strip = true  # Automatically strip symbols from the binary.
+
+[dependencies]
+axum = { version = "0.6.18", features = ["json"] }
+serde = { version = "1.0.167", features = ["derive"] }
+tokio = { version = "1.29.1", features = ["full"] }
+maud = { version = "*", features = ["axum"] }
+tower = { version = "0.4.13", features = ["util"] }
+hyper = { version = "0.14", features = ["full"] }
+tower-http = { version = "0.4.1", features = ["fs", "add-extension", "auth", "compression-full", "trace"] }
+sha2 = "0.10.7"
+hex = "0.4"
+futures = "0.3.28"
+comrak = "0.1"
+orgize = { git = "https://git.gmem.ca/arch/orgize.git", branch = "org-images" }
+clap = { version = "4.0", features = ["derive"] }
+serde_dhall = "0.12.1"
+frontmatter = "0.4.0"
+file-format = "0.18.0"
+rss = "2.0.6"
+time = { version = "0.3.28", features = ["parsing", "formatting", "macros"] }
+async-trait = "0.1.73"
+crossbeam = "0.8.2"
+rand = "0.8.5"
+prost = "0.12"
+
+[build-dependencies]
+prost-build = "0.12"

Dockerfile

@@ -0,0 +1,34 @@
+FROM rust:bookworm as builder
+
+WORKDIR /usr/src/app
+COPY . .
+
+RUN apt-get update -y && apt-get install -y ca-certificates protobuf-compiler && \
+	apt-get clean autoclean && \
+	apt-get autoremove --yes && \
+	rm -rf /var/lib/{apt,dpkg,cache,log}/
+
+# Will build and cache the binary and dependent crates in release mode
+RUN --mount=type=cache,target=/usr/local/cargo,from=rust:latest,source=/usr/local/cargo \
+    --mount=type=cache,target=target \
+    cargo build --release && cp ./target/release/gabrielsimmerdotcom ./gabrielsimmerdotcom
+
+# Runtime image
+FROM debian:bookworm-slim
+
+RUN apt-get update -y && apt-get install -y ca-certificates && \
+	apt-get clean autoclean && \
+	apt-get autoremove --yes && \
+	rm -rf /var/lib/{apt,dpkg,cache,log}/
+
+WORKDIR /app
+
+# Get compiled binaries from builder's cargo install directory
+COPY --from=builder /usr/src/app/gabrielsimmerdotcom /app/gabrielsimmerdotcom
+COPY --from=builder /usr/src/app/posts/ /app/posts/
+COPY --from=builder /usr/src/app/assets/ /app/assets/
+COPY --from=builder /usr/src/app/dhall/ /app/dhall
+
+EXPOSE 8080
+
+ENTRYPOINT ["/app/gabrielsimmerdotcom", "--bind", "0.0.0.0:8080"]

README.md

@@ -0,0 +1,3 @@
+# gabrielsimmer.com
+
+Overengineer the shit out of my personal site? Yes please.

build.rs

@@ -0,0 +1,6 @@
+use std::io::Result;
+
+fn main() -> Result<()> {
+    prost_build::compile_protos(&["protobuf/items.proto"], &["protobuf/"])?;
+    Ok(())
+}

dhall/projects.dhall

@@ -1,34 +1,21 @@
-let Url : Type =
-  { display_text : Text
-  , link : Text
-  }
+let types = ./types.dhall
 
-let Project : Type =
-  { name : Text
-  , description : Text
-  , link : Url
-  , languages : List Text
-  }
+let Project = types.Project
 
 let projects : List Project =
-  [ { name = "VRChat Prometheus Adapter"
-    , description = "Collect metrics from VRChat and VRCDN"
-    , link = { display_text = "git.gmem.ca/arch/vrchat-prometheus-adapter", link = "https://git.gmem.ca/arch/vrchat-prometheus-adapter" }
-    , languages = [ "Rust" ]
-    }
-  , { name = "GabrielSimmer.com"
+  [ { name = "GabrielSimmer.com"
     , description = "Overengineered personal site with on the fly page generation and tiered caching"
     , link = { display_text = "git.gmem.ca/arch/gabrielsimmer.com", link = "https://git.gmem.ca/arch/gabrielsimmer.com" }
     , languages = [ "Rust" ]
     }
   , { name = "My infrastructure"
-    , description = "Mostly Kubernetes manifests, with some Nix mixed in"
+    , description = "A mix of Terraform and Nix configurations keeping my lights on"
     , link = { display_text = "git.gmem.ca/arch/inra", link = "https://git.gmem.ca/arch/infra" }
-    , languages = [ "YAML", "Nix" ]
+    , languages = [ "Terraform", "Nix" ]
     }
   ,	{ name = "Minecraft Server Invites"
     , description = "Small service that generates links to Minecraft servers, whitelisting players"
-    , link = { display_text = "git.gmem.ca/arch/minecraft-server-invites", link = "https://git.gmem.ca/arch/minecraft-server-invites/" }
+    , link = { display_text = "gmem/minecraft-server-invites", link = "https://sr.ht/~gmem/minecraft-server-invites/" }
     , languages = [ "Go", "Svelte" ]
     }
   , { name = "artbybecki.com"
@@ -74,27 +61,7 @@ let projects : List Project =
   ]
 
 let experiments : List Project =
-  [ { name = "thirdrule"
-    , description = "Generate a random, surreal Discord server rule every 24 hours."
-    , link = { display_text = "git.gmem.ca/arch/thirdrule", link = "https://git.gmem.ca/arch/thirdrule" }
-    , languages = [ "JavaScript" ]
-    }
-  , { name = "friends-workers"
-    , description = "Collect and display buttons for friend's websites."
-    , link = { display_text = "git.gmem.ca/arch/friends-workers", link = "https://git.gmem.ca/arch/friends-workers" }
-    , languages = [ "JavaScript" ]
-    }
-  , { name = "daily-servo"
-    , description = "Generate snapshots of Servo-rendered webpages to track progress."
-    , link = { display_text = "git.gmem.ca/arch/daily-servo", link = "https://git.gmem.ca/arch/daily-servo" }
-    , languages = [ "JavaScript", "Shell" ]
-    }
-  , { name = "Discord bots"
-    , description = "Various single-purpose Discord bots."
-    , link = { display_text = "arch.dog/discord-bots", link = "https://arch.dog/discord-bots" }
-    , languages = [ "Misc." ]
-    }
-  , { name = "hue-webapp"
+  [ { name = "hue-webapp"
     , description = "Small web frontend and proxy for interacting with Hue lighting."
     , link = { display_text = "gmemstr/hue-webapp", link = "https://github.com/gmemstr/hue-webapp" }
     , languages = [ "Python" ]

dhall/types.dhall

@@ -0,0 +1,17 @@
+let Url : Type =
+  { display_text : Text
+  , link : Text
+  }
+
+
+let Project : Type =
+  { name : Text
+  , description : Text
+  , link : Url
+  , languages : List Text
+  }
+
+in
+  { Project
+  , Url
+  }

dhall/web-copy.dhall

@@ -0,0 +1,26 @@
+let types = ./types.dhall
+
+let Url = types.Url
+
+let subtexts : List Text =
+  [ "Infrastructure with Love"
+  , "My fourth Kubernetes cluster"
+  , "Overengineer my personal site?"
+  , "Mythical full stack engineer" 
+  ]
+
+let about : Text = ''
+Working as a Senior Platform Engineer in food delivery and logistics. Currently living in the United Kingdom. Working on some cool things for Furality. Playing around with various things in my free time.
+''
+
+let links : List Url =
+  [ { display_text = "Fediverse"
+  	, link = "https://floofy.tech/@arch"
+	}
+  ]
+
+in
+  { about
+  , subtexts
+  , links
+  }

flake.lock

@@ -1,18 +1,36 @@
 {
   "nodes": {
+    "advisory-db": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1696104803,
+        "narHash": "sha256-xdPl4PHzWaisHy4hQexpHJlx/3xAkCZrOFtEv3ygXco=",
+        "owner": "rustsec",
+        "repo": "advisory-db",
+        "rev": "46754ce9372c13a7e9a05cf7d812fadfdca9cf57",
+        "type": "github"
+      },
+      "original": {
+        "owner": "rustsec",
+        "repo": "advisory-db",
+        "type": "github"
+      }
+    },
     "crane": {
       "inputs": {
+        "flake-compat": "flake-compat",
+        "flake-utils": "flake-utils",
         "nixpkgs": [
-          "freight",
           "nixpkgs"
-        ]
+        ],
+        "rust-overlay": "rust-overlay"
       },
       "locked": {
-        "lastModified": 1712180168,
-        "narHash": "sha256-sYe00cK+kKnQlVo1wUIZ5rZl9x8/r3djShUqNgfjnM4=",
+        "lastModified": 1696266955,
+        "narHash": "sha256-GhaBeBWwejBTzBQl803x7iUXQ6GGUZgBxz+qyk1E3v4=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "06a9ff255c1681299a87191c2725d9d579f28b82",
+        "rev": "581245bf1233d6f621ce3b6cb99224a948c3a37f",
         "type": "github"
       },
       "original": {
@@ -23,20 +41,39 @@
     },
     "fenix": {
       "inputs": {
-        "nixpkgs": "nixpkgs",
-        "rust-analyzer-src": "rust-analyzer-src"
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-analyzer-src": []
       },
       "locked": {
-        "lastModified": 1712211755,
-        "narHash": "sha256-KIJA4OvXFDXEeu7wstFDCxqZEfjaPQIowpzNww48TUw=",
+        "lastModified": 1696314121,
+        "narHash": "sha256-Dd0xm92D6cQ0c46uTMzBy7Zq9tyHScArpJtkzT87L4E=",
         "owner": "nix-community",
         "repo": "fenix",
-        "rev": "39763c6e23a8423af316b85a74bad0cc5bc88d86",
+        "rev": "c543df32930d075f807fc0b00c3101bc6a3b163d",
         "type": "github"
       },
       "original": {
-        "id": "fenix",
-        "type": "indirect"
+        "owner": "nix-community",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1673956053,
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
       }
     },
     "flake-utils": {
@@ -44,11 +81,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1710146030,
-        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "lastModified": 1694529238,
+        "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "rev": "ff7b65b44d01cf9ba6a71320833626af21126384",
         "type": "github"
       },
       "original": {
@@ -57,86 +94,71 @@
         "type": "github"
       }
     },
-    "freight": {
+    "flake-utils_2": {
       "inputs": {
-        "crane": "crane",
-        "fenix": "fenix",
-        "flake-utils": [
-          "flake-utils"
-        ],
-        "nixpkgs": [
-          "nixpkgs"
-        ],
-        "workers-rs": "workers-rs"
+        "systems": "systems_2"
       },
       "locked": {
-        "lastModified": 1712226020,
-        "narHash": "sha256-SXaBtGDwCWG74QeONSipOJYAU0iz75fJIlDMptb0bKM=",
-        "owner": "gmemstr",
-        "repo": "freight",
-        "rev": "a3a2fd9e863622c7bc1472bcd8ef100e5fc763a0",
+        "lastModified": 1694529238,
+        "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "ff7b65b44d01cf9ba6a71320833626af21126384",
         "type": "github"
       },
       "original": {
-        "owner": "gmemstr",
-        "ref": "patch-1",
-        "repo": "freight",
+        "owner": "numtide",
+        "repo": "flake-utils",
         "type": "github"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1712122226,
-        "narHash": "sha256-pmgwKs8Thu1WETMqCrWUm0CkN1nmCKX3b51+EXsAZyY=",
-        "owner": "nixos",
+        "lastModified": 1696261572,
+        "narHash": "sha256-s8TtSYJ1LBpuITXjbPLUPyxzAKw35LhETcajJjCS5f0=",
+        "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "08b9151ed40350725eb40b1fe96b0b86304a654b",
+        "rev": "0c7ffbc66e6d78c50c38e717ec91a2a14e0622fb",
         "type": "github"
       },
       "original": {
-        "owner": "nixos",
-        "ref": "nixos-unstable",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs_2": {
-      "locked": {
-        "lastModified": 1712122226,
-        "narHash": "sha256-pmgwKs8Thu1WETMqCrWUm0CkN1nmCKX3b51+EXsAZyY=",
-        "owner": "nixos",
-        "repo": "nixpkgs",
-        "rev": "08b9151ed40350725eb40b1fe96b0b86304a654b",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nixos",
-        "ref": "nixos-unstable",
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
     "root": {
       "inputs": {
-        "flake-utils": "flake-utils",
-        "freight": "freight",
-        "nixpkgs": "nixpkgs_2"
+        "advisory-db": "advisory-db",
+        "crane": "crane",
+        "fenix": "fenix",
+        "flake-utils": "flake-utils_2",
+        "nixpkgs": "nixpkgs"
       }
     },
-    "rust-analyzer-src": {
-      "flake": false,
+    "rust-overlay": {
+      "inputs": {
+        "flake-utils": [
+          "crane",
+          "flake-utils"
+        ],
+        "nixpkgs": [
+          "crane",
+          "nixpkgs"
+        ]
+      },
       "locked": {
-        "lastModified": 1712156296,
-        "narHash": "sha256-St7ZQrkrr5lmQX9wC1ZJAFxL8W7alswnyZk9d1se3Us=",
-        "owner": "rust-lang",
-        "repo": "rust-analyzer",
-        "rev": "8e581ac348e223488622f4d3003cb2bd412bf27e",
+        "lastModified": 1695003086,
+        "narHash": "sha256-d1/ZKuBRpxifmUf7FaedCqhy0lyVbqj44Oc2s+P5bdA=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "b87a14abea512d956f0b89d0d8a1e9b41f3e20ff",
         "type": "github"
       },
       "original": {
-        "owner": "rust-lang",
-        "ref": "nightly",
-        "repo": "rust-analyzer",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
         "type": "github"
       }
     },
@@ -155,19 +177,18 @@
         "type": "github"
       }
     },
-    "workers-rs": {
-      "flake": false,
+    "systems_2": {
       "locked": {
-        "lastModified": 1712083113,
-        "narHash": "sha256-k8F8iX7kS+Jfz22ge84P7DoL1PAM+uuB/5uhjvzvY5w=",
-        "owner": "cloudflare",
-        "repo": "workers-rs",
-        "rev": "c1805b65285e4d0fa16c1496ea7d7052b44b4655",
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
         "type": "github"
       },
       "original": {
-        "owner": "cloudflare",
-        "repo": "workers-rs",
+        "owner": "nix-systems",
+        "repo": "default",
         "type": "github"
       }
     }

flake.nix

@@ -1,57 +1,153 @@
 {
+  description = "Build a cargo project";
+
   inputs = {
-    nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
-    flake-utils.url = "github:numtide/flake-utils";
-    freight = {
-      url = "github:gmemstr/freight/patch-1";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+
+    crane = {
+      url = "github:ipetkov/crane";
       inputs.nixpkgs.follows = "nixpkgs";
-      inputs.flake-utils.follows = "flake-utils";
+    };
+
+    fenix = {
+      url = "github:nix-community/fenix";
+      inputs.nixpkgs.follows = "nixpkgs";
+      inputs.rust-analyzer-src.follows = "";
+    };
+
+    flake-utils.url = "github:numtide/flake-utils";
+
+    advisory-db = {
+      url = "github:rustsec/advisory-db";
+      flake = false;
     };
   };
 
-  outputs = {
-    nixpkgs,
-      flake-utils,
-      freight,
-      ...
-  }:
-    flake-utils.lib.eachDefaultSystem (
-      system:
+  outputs = { self, nixpkgs, crane, fenix, flake-utils, advisory-db, ... }:
+    flake-utils.lib.eachDefaultSystem (system:
       let
         pkgs = import nixpkgs {
           inherit system;
         };
-        inherit (pkgs) lib; in
+
+        inherit (pkgs) lib;
+
+        craneLib = crane.lib.${system};
+        src = lib.cleanSourceWith {
+          src = craneLib.path ./.; # The original, unfiltered source
+          filter = path: type:
+            let
+              protoFilter = path: _type: builtins.match ".*proto$" path != null;
+              protoOrCargo = path: type:
+                (protoFilter path type) || (craneLib.filterCargoSources path type);
+            in
+              protoOrCargo path type;
+        };
+        # Common arguments can be set here to avoid repeating them later
+        commonArgs = {
+          inherit src;
+          buildInputs = [
+            pkgs.sqlite
+            pkgs.pkg-config
+            pkgs.openssl
+            pkgs.protobuf
+          ] ++ lib.optionals pkgs.stdenv.isDarwin [
+            # Additional darwin specific inputs can be set here
+            pkgs.libiconv
+          ];
+        };
+
+        craneLibLLvmTools = craneLib.overrideToolchain
+          (fenix.packages.${system}.complete.withComponents [
+            "cargo"
+            "llvm-tools"
+            "rustc"
+          ]);
+
+        # Build *just* the cargo dependencies, so we can reuse
+        # all of that work (e.g. via cachix) when running in CI
+        cargoArtifacts = craneLib.buildDepsOnly commonArgs;
+
+        # Build the actual crate itself, reusing the dependency
+        # artifacts from above.
+        
+        my-crate = craneLib.buildPackage (commonArgs // {
+          inherit cargoArtifacts;
+        });
+      in
         {
-          packages = rec {
-            web = freight.lib.${system}.mkWorker {
-              pname = "web";
-              version = "0.1.0";
-              strictDeps = true;
-              workspace = true;
-              src = ./.;
+          checks = {
+            # Build the crate as part of `nix flake check` for convenience
+            inherit my-crate;
+
+            # Run clippy (and deny all warnings) on the crate source,
+            # again, resuing the dependency artifacts from above.
+            #
+            # Note that this is done as a separate derivation so that
+            # we can block the CI if there are issues here, but not
+            # prevent downstream consumers from building our crate by itself.
+            my-crate-clippy = craneLib.cargoClippy (commonArgs // {
+              inherit cargoArtifacts;
+              cargoClippyExtraArgs = "--all-targets -- --deny warnings";
+            });
+
+            my-crate-doc = craneLib.cargoDoc (commonArgs // {
+              inherit cargoArtifacts;
+            });
+
+            # Check formatting
+            my-crate-fmt = craneLib.cargoFmt {
+              inherit src;
             };
-            queues = freight.lib.${system}.mkWorker {
-              pname = "queues";
-              version = "0.1.0";
-              strictDeps = true;
-              workspace = true;
-              src = ./.;
+
+            # Audit dependencies
+            my-crate-audit = craneLib.cargoAudit {
+              inherit src advisory-db;
             };
-            default = web;
+
+            # Run tests with cargo-nextest
+            # Consider setting `doCheck = false` on `my-crate` if you do not want
+            # the tests to run twice
+            my-crate-nextest = craneLib.cargoNextest (commonArgs // {
+              inherit cargoArtifacts;
+              partitions = 1;
+              partitionType = "count";
+            });
+          } // lib.optionalAttrs (system == "x86_64-linux") {
+            # NB: cargo-tarpaulin only supports x86_64 systems
+            # Check code coverage (note: this will not upload coverage anywhere)
+            my-crate-coverage = craneLib.cargoTarpaulin (commonArgs // {
+              inherit cargoArtifacts;
+            });
           };
+
+          packages = {
+            default = my-crate;
+            my-crate-llvm-coverage = craneLibLLvmTools.cargoLlvmCov (commonArgs // {
+              inherit cargoArtifacts;
+            });
+          };
+
+          apps.default = flake-utils.lib.mkApp {
+            drv = my-crate;
+          };
+
           devShells.default = pkgs.mkShell {
+            inputsFrom = builtins.attrValues self.checks.${system};
+
+            # Additional dev-shell environment variables can be set directly
+            # MY_CUSTOM_DEVELOPMENT_VAR = "something else";
+
+            # Extra inputs can be added here
             nativeBuildInputs = with pkgs; [
               cargo
               rustc
               rust-analyzer
-              nodePackages_latest.wrangler
-              openssl
-              pkg-config
-              cargo-bloat
-              cargo-binutils
+              sqlite
+              sqlx-cli
+              flyctl
+              cargo-flamegraph
             ];
           };
-        }
-    );
+        });
 }

fly.toml

@@ -0,0 +1,14 @@
+# fly.toml app configuration file generated for gabrielsimmerdotcom on 2023-07-19T21:27:38+01:00
+#
+# See https://fly.io/docs/reference/configuration/ for information about how to use this file.
+#
+
+app = "gabrielsimmerdotcom"
+primary_region = "lhr"
+
+[http_service]
+  internal_port = 8080
+  force_https = true
+  auto_stop_machines = false
+  auto_start_machines = true
+  processes = ["app"]

k6/smoketest.js

@@ -0,0 +1,12 @@
+import http from 'k6/http';
+import { check, sleep} from 'k6';
+
+export const options = {
+  vus: 3, // Key for Smoke test. Keep it at 2, 3, max 5 VUs
+  duration: '1m', // This can be shorter or just a few iterations
+};
+
+export default () => {
+  const urlRes = http.get('https://gabrielsimmer.com');
+  sleep(1);
+};

migrations/20230719160618_initial.sql

@@ -0,0 +1,9 @@
+-- Add migration script here
+create table cached (
+	   route text not null unique,
+	   cached int not null,
+	   content_type text not null,
+	   content text not null
+);
+
+create unique index path_unique_idx on cached (lower(route));

posts/dns-can-be-a-nightmare.org

@@ -1,29 +0,0 @@
-#+title: DNS can be a nightmare
-#+date: 2024-04-29
-
-I'm a simple dog. I like my coffee black, my music at a reasonable volume, and my networking fast. Lately, I'm been looking to further that third one by way of reducing network latencies and calls where I can. While poking around my NextDNS dashboard, as I do from time to time, I realised that /wow/, my devices are making a lot of queries to NextDNS servers. Surely I can reduce the amount of time it takes to resolve a DNS query by caching those responses closer to my devices?
-
-In my line of work, we talk a lot about "running at the edge", as close to the people using the service as possible. What's closer than physically beside my desk, on my home subnet? Therefore, I set out with a simple task - build a shared DNS cache for my home network.
-
-Conceptually, this is not an unusual task. Routers usually do this by default, plenty of clients cache DNS queries, and so on. So I was fairly confident it would be easy to do - couple LXC containers running in Proxmox, connected to Tailscale, and I'd be golden. The idea was to run two dnsmasq instances, one at home and one in the Floofy.tech Proxmox server (for redundancy more than anything), that all the devices on my Tailscale network would use for DNS queries. Those instances would forward to the NextDNS profile I have setup specifically for those devices (with several DNS rewrites to point to Tailscale IPs) and then cache the response so that subsequent requests from the same or other devices would be much faster.
-
-I'll admit, the setup of dnsmasq itself was very smooth. All I had to do was grab the configuration NextDNS generates for me and pop it into =/etc/dnsmasq.d/nextdns.conf=, then tweak the configuration of the cache size and interface it should listen on. dnsmasq itself is intended to be run, from what I can tell anyways, on a single device and only handle the queries from that device, and I found that the Debian package installs the init.d script with this in mind. Once I'd configured it, I pointed my Tailscale DNS settings to the machine's Tailnet IP, and it was off to the races! The results were pretty immediate (if somewhat imperceptible to the human eye), and I quickly set about setting up the second LXC in the Floofy.tech Proxmox server. I was happy.
-
-[[https://cdn.gabrielsimmer.com/images/dns-query-time.png]]
-/DNS resolution time dropped dramatically, time in ms/
-
-My hubris was met not with immediate consequence, but a slow, creeping one. I found that the Floofy.tech LXC was being tempermental, losing its network connection completely. It was later in the day, so I opted to just remove it from the Tailscale DNS settings and figure it out in the morning.
-
-I woke up to my personal infrastructure on fire, but quietly. At some point during the night, the dnsmasq LXC on my personal Proxmox box also fell over, this time a little more dramatically, knocking the entire host offline. Groggy, I rebooted the machine and set about the rest of the recovery, disabling the Tailscale DNS machinery on my desktop and set about figuring out what happened (after I switched off "override DNS" in Tailscale's dashboard). As far as I could tell, it had nothing to do with dnsmasq, but rather the way the =/dev/tun= device needs to be mounted into the LXC container for Tailscale to function. My dreams of a lightweight container running the cache in tatters, I swapped to using virtual machines on both boxes.
-
-Everything going wrong once again reminded me how badly things can end up if your DNS goes down or misbehaves. The result isn't always immediate, or obvious. It creeps up on you as caches expire, slowly cutting off your ability to connect to websites. This can be especially frustrating when the very thing you use to manage DNS configuration is a remote website.
-
-Anyways! The result is fairly reasonable. I've cut down DNS resolution times a fairly significant amount, which was really the end goal. I also configured my router's DNS properly to route DNS requests through its own dnsmasq instance (no custom built router... yet...) so generally devices on my network get faster responses, even if I have less control over how it works. DNS rewriting with NextDNS still works, since the caches will query it for answers.
-
-[[https://cdn.gabrielsimmer.com/images/dnsmasq-dashboard.png]]
-/My dnsmasq Grafana dashboard using [[https://git.gmem.ca/arch/dnsmasq_exporter][dnsmasq_exporter]]/
-
-[[https://cdn.gabrielsimmer.com/images/dns-nextdns-drop.png]]
-/Grafana dashboard showing a substantial drop in queries to NextDNS/
-
-Now we'll see how long it lasts.

posts/mastodon-redis-sentinel.org

@@ -1,18 +0,0 @@
-#+title: Mastodon Redis Sentinel
-#+date: 2024-03-19
-
-In my quest to build stupid overkill infrastructure, I've included the Floofy.tech Mastodon instance in my endevour (it's okay, my co-owner has too). Part of this has been making sure that we have redundancy where possible, whether it be a high-availability Kubernetes cluster or multiple instances of certain networking portions to make sure traffic can flow while we work on upgrades or the like. One thing that has been missing is a piece rather critical to Mastodon and its queue runner (Sidekiq) - Redis. Redis is used by Mastodon as both a cache as well as a persistant store for Sidekiq jobs, and is required to be up and functional for an instance to operate. However, it didn't have a mechanism to account for a Redis Cluster or Redis Sentinel setup, and would have required some work with another load balancer like HAProxy to get working.
-
-Some background on Redis high availability - there are two options, clustering or Sentinel. Clustering is a more traditional HA approach - you have several Redis instances working together as one homogeneous unit, redirecting clients to nodes that contain to data they need to read or write (notably, not /proxying/ the requests, but rather telling the client to retry with another machine). Data is sharded in a way where multiple Redis instances can contain the same data, so if some go down the data is still available. While this is great for performance and scaling up Redis, it comes at the cost of consistency - a write to one instance does not instantly propogate to another (but does relatively quickly). We also have Redis Sentinel, where a set of one or more Sentinel services connects to two or more Redis instances and monitors them. One Redis instance is elected the initial "master" (this is the term Redis uses, for simplicity it's the term I'll be using) and the Sentinel instance(s) provide this information to clients. Clients can check in with the Sentinel instances for a new master if they lose connection with the Redis instance, and write to the master instance will propogate to other instances monitored by the Sentinel instance(s). These secondary Redis instances are configured as replicas of the master, using the same replication method as a Redis Cluster. /However/, since we only ever read and write from one Redis instance, we worry much less about lost data. There /is/ still an opportunity for this to happen if the master drops out unexpectedly, but there are ways to reduce the window of opportunity for this. Because of this key difference, Sentinel is suitable for use with Sidekiq and other backends that require stronger consistency than what a Redis Cluster can offer.
-
-With all that said, Mastodon did not support Redis Sentinel (or Redis Cluster). While Ruby on Rails, Sidekiq, and ioredis (used for the websocket server) all supported it, there were no configuration options exposed to enable it in the app - so I took it upon myself to make [[https://github.com/mastodon/mastodon/pull/26571][the pull request]] enabling it. While writing this it's still a bit of a work in progress, but we'll get to the current state of the PR in a moment.
-
-This is really my first time using Ruby and Ruby on Rails, and initial impression was not one of dislike. At least on the Ruby side - for frameworks like Rails or Django, I tend to be a little less happy about working with due to the amount of magic they attempt to do for a smoother developer experience. Thankfully the changes that had to be made were fairly small and contained, and the majority of my time was taken up testing behaviour to make sure it did what I expected. The biggest point of concern was time to recovery - if the master Redis instance goes down, how long until a secondary is promoted and how long after that does the Mastodon instance pick up the change? The answer is "it's configurable" and also depends on the failure mode - a graceful shutdown will trigger a graceful handover, while a sudden loss will take a couple minutes by default to swap over.
-
-The pull request still needs some work, and a bit of code cleanup, but it is functional, with some caveats. The documentation around using Redis Sentinel with redis-rb is a little sparse, and the biggest question mark is around DNS resolution for determining Redis Sentinel servers to connect to. It's something I'm currently working on, but in the meantime...
-
-I deployed the patch to Floofy.tech. Something something, don't test in production, but frankly the only way to properly test this is with a setup like Floofy.tech, with real traffic. There was a minor issue where the Mastodon deployments couldn't find the Sentinels to connect to, but this was solved by a restart of the pods - I can't recall exactly what went wrong in this situation, but it was likely applying the changes needed out of order. Those changes included increasing the number of replicas for our Redis Helm Chart, pointing the Mastodon deployments to our custom image (glitch-soc Mastodon fork with the Redis Sentinel patch applied), and adding the configuration to Mastodon's environment. Once the pull request is closer to completion I'll likely update the pull request I have for the docs with instructions for migrating from a single Redis insance to a Sentinel setup, but in the meantime those are my cliff notes.
-
-Despite all the testing I did during the pull request process, I decided to do a quick "cut test" with Floofy.tech, deliberately simulating various failure modes for our Redis instances and watching the failover. This included powering off a Kubernetes node unexpectedly, powering it off gracefully, and deleting the pod. In all cases, Redis Sentinel performed as expected, where the unexpected shutdowns forced a failover after a short delay and expected shutdowns performed a proper handover. Amusingly, the nodes rebooted so quickly that a simulated unexpected reboot didn't have much of an impact - it had to be force stopped instead. Overall, though, I was satisfied with the results and with my findings in hand I called it a day.
-
-While there is still some work to do on the pull request, and patch, I'm overall happy with the current state of it and feel comfortable continuing to run it on Floofy.tech. I wouldn't /discourage/ using it on your own instance, but just be mindful of the potential risks. If using it, I also highly recommend using it only for the Sidekiq and streaming portions of Mastodon, and create a dedicated Redis instance for caching.

posts/migrating-mastodon-s3-providers.org

@@ -1,16 +0,0 @@
-#+title: Migrating Mastodon S3 Providers
-#+date: 2024-03-08
-
-As part of some recent work Kakious and I have been doing on the Mastodon instance we run, Floofy.tech, we were looking to cut costs while also improving performance. Part of that was migrating to new hardware using Proxmox rather than ESXi, and that migration was fairly unremarkable - we're still running everything in Kubernetes across multiple VMs, so there isn't really much to note (except we also moved to using Talos over k3s - I will have a longer post about that at some point). Another part of this was moving storage providers for the instance. We were using DigitalOcean Spaces, which served us well, but the pricing left quite a bit on the table. For $5/month, you get 250GiB of storage and 1TB of egress, with $0.02/GiB stored and $0.01GiB for egress after that. Our instance fell comfortably in this bucket, but /very/ comfortably, to the point we would certainly save money going elsewhere. Being employed at Cloudflare, and already having a decent setup for Floofy there, we turned to the R2 offering. With no egress costs and less than 100GiB stored (on average - depends how busy the fediverse is!), we shouldn't be paying anywhere near $5/month, and we're only paying for storage since egress is free and heavily cached.
-
-So! With that decided, it was time to figure out our migration path. The plan was simple - using rclone, setup two remotes on a temporary virtual machine (or LXC container, in this case), and run a bulk transfer overnight. Once complete, we run one more sync then quickly swap the configuration on the Mastodon instance. The window of time between the last sync and Mastodon instances picking up the new configuration should be small enough that we don't miss any new media. Finally, we swap over the DNS to point to our R2 bucket, which should update quickly as the DNS was already proxied through Cloudflare.
-
-Setup of the container was straightforward - we grabbed a Debian 12 image, installed rclone, and then setup two remotes. One pointed to our existing DigitalOcean Spaces bucket (=digitalocean-space=) and the other our new Cloudflare R2 bucket (=cloudflare-r2=). A quick =rclone lsd= on both remotes to confirm connection later, and then a dry run sync or two to verify, we were ready to go. I loaded up tmux, hit go, and waited.
-
-It was going smoothly, until the container decided to drop its internet connection. I'm still not sure what caused this but after running =dhclient= it was fine again. The sync went off without a hitch otherwise.
-
-When I woke up, it was time to run another sync then make the configuration change. I'd already changed the values in our Kubernetes state repository, so it was just a case of pushing it to GitHub and letting ArgoCD sync everything up. First, I reran the =rclone sync= to ensure that anything new was synced over, then quickly pushed the configuration up. It took about a minute to cycle the pods to the new configuration, at which point I removed the DNS record pointing to DigitalOcean and swapped it over to the Cloudflare R2 bucket. Done!
-
-I genuinely expected this to be more difficult, but it really was that easy. This process would work for any rclone-compatible storage provider, of which there are many, so I'd feel pretty comfortable recommending this to others. Depending how busy your instance is, it may be worth doing a final =rclone copy= (which copies new files but doesn't delete from the target) to catch and stragglers after the configuration change, and depending on your DNS setup you may need to modify the time-to-live values ahead of the migration, but we didn't really hit those caveats.
-
-Hopefully this is helpful to others - if you have any questions, feel free to poke me on the federated universe [[https://floofy.tech/@arch][@arch@floofy.tech]].

posts/nitter-is-not-dead.org

@@ -1,36 +0,0 @@
-#+title: Nitter is Not Dead
-#+date: 2024-03-10
-
-A quick one for you all today.
-
-The maintainer of Nitter said, not long ago, that [[https://github.com/zedeus/nitter/issues/1155#issuecomment-1913361757]["Nitter is dead."]]. This isn't totally inaccurate, as Twitter/X has shut down huge swaths of their public API, especially ones Nitter relied on, making it very, very difficult to properly run a public instance. But private instances are still viable with a bit of tinkering.
-
-The first step is cloning down the Nitter source code using Git. You'll then need to checkout the =guest_accounts= branch, and optionally build the Docker image for it. The important piece is ensuring you're running that branch's code wherever you're deploying.
-
-Next, you'll need some tokens to feed into Nitter. Thankfully, someone has already written some Python to do this for us, which you can find [[https://github.com/zedeus/nitter/issues/983#issuecomment-1914616663][here]]. Fill out the username and password, then run it. This will return a few JSON payloads, but the one you want is at the very end (you will also want to convert the single quotes to double quotes). Create a new file alongside your =nitter.conf= file named =guest_accounts.json= with the JSON in an array:
-
-#+begin_src json
-[
-    {
-        "user": {
-            "id": 123,
-            "id_str": "...",
-            "name": "...",
-            "screen_name": "..."
-        },
-        "next_link": {
-            "link_type": "subtask",
-            "link_id": "next_link",
-            "subtask_id": "SuccessExit"
-        },
-        "oauth_token": "...",
-        "oauth_token_secret": "...",
-        "known_device_token": "...",
-        "attribution_event": "login"
-    }
-]
-#+end_src
-
-Finally, run Nitter! Make sure you have a Redis instance setup for it for caching, and you should be good to go. You can add as many accounts to the =guest_accounts.json= file as you need, which Nitter will cycle through as needed to spread out the requests and rate limiting. Using a proxy is probably also recommended, but it's likely Twitter/X is blocking many of those at this point.
-
-I expect Twitter/X to eventually crack down further on this, but for the time being we can continue to enjoy the nicer and more privacy concious interface Nitter provides.

posts/porting-to-workers.org

@@ -1,31 +0,0 @@
-#+title: Porting to Workers
-#+date: 2024-01-28
-
-This website is now using Cloudflare Workers!
-
-I went a little bit crazy over the weekend and decided to finally make the move from the three-region Fly.io deployment to Workers that run at the edge. This does mean I'm abandoning the gossip protocol based content cache dissemination implementation I was working on for the Fly.io deployment, but I still think the idea is valuable, and I'll try to find something else to add it to.
-
-Choosing how to build this site on Workers was a bit tricky. My goal continues to be to overengineer what it ostensibly a static site for fun and negative profit. Thus, I completely ignored Cloudflare Pages. I wanted to keep a tiered caching strategy that I tried in the initial version of the Fly.io version, and it would be easier given I could leverage Cloudflare's caching systems. Ideally I want to regenerate pages from source when things change, but I also have to pay attention to how long it takes to execute a function when called. With Fly.io, I wanted fast page generation, but I didn't really care hold long it took since the CPU time wasn't limited in any way. Another limitation is that Workers don't really have a filesystem, so I'd have to leverage the various offerings available to store files. With Fly.io, I built the Docker image with the posts bundled in, which was great for speed but felt a little less than ideal since there was no clear separation between publishing a post and deploying changes to the backend.
-
-So first, post storage! With Worker size limits, I decided to go with storing posts in their raw (=.org= or =.md=) format in an R2 bucket. While R2 is good for scaling storage, it does break my "live on the edge" mindset. So I needed a way to distribute the content to the edge, avoiding the Workers needing to reach into a region further away.
-
-After some consideration, I scrapped the idea of generating and storing the result for other Workers on the fly and looked at the Queue option instead. The plan was to pre-render the content and store it somewhere (more on that later) so I can very quickly render content in the background when something is published. When a file is pushed to R2, I can fire of a webhook that queues up the new or changed files for rendering and storing on the edge. It does seem to introduce a little more latency when it comes to publishing content, but in reality it's faster because it doesn't require me to rebuild, push, and restart a container image.
-
-Where to store the rendered content stuck with me for a bit. Initially I wanted to go with KV, since it seemed it would be faster, but I found after some experimentation it was substantially slower since there's no way to easily sort the keys based on content without reading /everything/ into memory and then sorting during Worker execution. Thankfully, I could reach for a real database, and created a D1 instance to hold a single table with the posts. It being SQLite based, I can just use SQL for the queries and take advantage of much more optimised codepaths for sorting or fetching the data I actually need. While D1 doesn't currently replicate, it will be a huge speed boost when it is!
-
-/Note: this section originally said that D1 replicates. I was then told and disovered this is not the case at the moment. Whoops./
-
-The workflow thus far is
-
- 1. A post is pushed to R2
- 2. A webhook is sent to a Worker (not by R2)
- 3. The worker fetches the list of files from R2 and queues them for "indexing"
- 4. Workers are executed to consume the queue, rendering the files and storing them in D1
-
-The final piece is telling the Worker it can cache all the responses in Cloudflare's cache, and we're all set! Each response is cached for 4 hours before a Worker has to be hit to fetch the content from D1 again.
-
-Of course, it wasn't all smooth sailing, which was mildly expected since D1 and Queues are both still in beta, and I'm using Rust, which, while having first party support through the [[https://github.com/cloudflare/workers-rs][workers-rs]] library, has some rough edges on Workers. The biggest one for my implementation is the inability to run the Worker locally when Queues are in use. Generally speaking my local development flow for this project needs improvement, and will be worked on, but the inability to run it locally with Queues (or even with =--remote= at the moment) makes iterating slow. I do believe this will improve over time, and when I hit blockers I plan to contribute back to the workers-rs library, but what I've needed to use works great.
-
-[[https://cdn.gabrielsimmer.com/images/workers-publish-workflow.png]]
-
-As per usual, I'm planning to continue iterating on the site, and hopefully make it faster and smaller over time. I do of course have the source available [[https://git.gmem.ca/arch/gabrielsimmer.com/src/branch/workers][in my Forgejo repo]], although there are still a few thing I need to do like implementing the actual CI/CD for publishing and webhooking. Feedback is welcome of course! Feel free to poke me [[https://floofy.tech/@arch][on the federated universe]].

posts/stop-scraping-git-forge.org

@@ -1,26 +0,0 @@
-#+title: Stop Scraping my Git Forge
-#+date: 2024-06-03
-
-Amazonbot, please. It's too much.
-
-I recently decided to dig into the webserver analytics for my [[https://git.gmem.ca][self-hosted Git forge]], which are provided by Cloudflare through my use of Tunnels to expose it to the wider internet. While generally I post my code under open source licenses to others can easily use it, learn from it, modify it, whatever they like, I /do not/ like the idea that large corporations are taking this same code and putting it into a black box that is a machine learning model.
-
-How this affects my use of the licenses I open source my code under, I'm unsure.
-
-Corporations stealing, or using work without permission, for their machine learning models has been a discussion for a long while at this point. In general, I side with the creators or artists having their work taken. While I have used various machine learning "things" in the past, from ChatGPT to Copilot (the code one, not Copilot+) to Stable Diffusion, it was mostly from the standpoint of curiousity - see what it could do, but not use whatever it spat out in any serious capacity. Text generation models were the most useful in my case. Organising thought or recalling specific terminology/techniques could sometimes be tricky, and being able to rubber duck with it proved handy as a springboard to go do "serious" research with a search engine or technical documents. The key here, of course, being that what the text models gave me /was not taken at face value/, and instead used to reorient myself in the right direction. Stable Diffusion is an entirely different thing, and being friends with a number of artists I was generally uncomfortable using it even if I was getting crappy results.
-
-But I digress, this isn't necessarily a post about my general feelings on machine learning. This is specifically about how corporations are gathering this data.
-
-Looking at my analytics, it isn't uncommon to see some crawlers going through various pages to index them, and I'm usually okay with it since it can help discoverability for the purposes of learning how systems are put together (I'm much less worried about people actually /using/ my software, although it is always a joy when someone does). But I saw a *massive* uptick in the amount of data (many gigabytes over a few days) that a certain =Amazonbot= (=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)=) was pulling from =git.gmem.ca= and decided to take a slightly closer look. Quicky checking the link included in the user agent, it appears it's used for Alexa related queries, but [[https://radar.cloudflare.com/traffic/verified-bots][Cloudflare Radar]] lists it as an "AI Crawler", which makes me suspect it's also feeding into whatever machine learning models they're building at AWS. This suspicion was further from the sheer amount of data and depth the crawler was pulling, pulling individual commits for projects (among other things). This did not impress me.
-
-While I'm specifically calling out =Amazonbot= here, it wasn't the only crawler. The difference being it was the most notable crawler given the volume of requests. That said, the other crawlers didn't have any business crawling my git forge either, so...
-
-I blocked them.
-
-[[https://cdn.gabrielsimmer.com/images/waf-rule-blocks.png]]
-
-One WAF rule later, and I have all "Known bots" blocked from visiting my git forge. I was inclined to block IP ranges or ASNs, but ultimately decided against it. I generally believe that information should be freely accessible to any individual who seeks it out, but corporations are not individuals, and while I can easily handle the amount of traffic (given I had to look at logs to notice it should tell you something), it still places an unexpected and unecessary amount of strain on my infrastructure.
-
-[[https://cdn.gabrielsimmer.com/images/waf-rule.png]]
-
-As of writing, the crawlers are still trying to crawl, and the WAF rule has blocked ~25,000 requests since I put the block in place (June 2nd 2024 5:15PM BST - June 3rd 2024 2:30PM BST). I'm resting a bit easier now, and would encourage others to look at their own logs to figure out whether their own instances are being scraped as well.

protobuf/items.proto

@@ -0,0 +1,51 @@
+syntax = "proto3";
+
+package gabrielsimmerdotcom.gossip;
+
+message Ping {}
+
+message PingRequest {
+  string peer = 1;
+}
+
+enum Status {
+  PEER_UNKNOWN = 0;
+  PEER_ALIVE = 1;
+  PEER_SUSPECT = 2;
+  PEER_CONFIRM = 3;
+}
+
+// { "127.0.0.1:1337", confirm }
+message Membership {
+  string peer = 1;
+  Status status = 2;
+  int64 inc = 3;
+}
+
+message NewCache {
+  string key = 1;
+  string content = 2;
+  string content_type = 3;
+}
+
+message Cache {
+  string key = 1;
+  string content = 2;
+  string content_type = 3;
+  int64 timestamp = 4;
+}
+
+message Payload {
+  string peer = 1;
+  oneof msg {
+	Cache cache = 2;
+	Ping ping = 3;
+	Ack ack = 4;
+	Membership membership = 5;
+	NewCache new_cache = 6;
+  }
+}
+
+message Ack {
+  
+}

queues/Cargo.toml

@@ -1,17 +0,0 @@
-[package]
-name = "queues"
-version = "0.1.0"
-edition = "2021"
-
-[lib]
-crate-type = ["cdylib", "rlib"]
-
-[dependencies]
-shared = { path = "../shared" }
-worker = { version = "0.1.0", features = ["queue", "d1"] }
-serde = { version = "1.0.167", features = ["derive"] }
-
-[profile.release]
-lto = true
-strip = true
-codegen-units = 1

queues/src/lib.rs

@@ -1,49 +0,0 @@
-use shared::blog_post;
-use worker::*;
-use serde::Deserialize;
-
-#[allow(dead_code)]
-#[derive(Deserialize, Debug)]
-struct R2Object {
-    key: String,
-    size: u64,
-    #[serde(rename = "eTag")]
-    e_tag: String,
-}
-
-#[allow(dead_code)]
-#[derive(Debug, Deserialize)]
-struct R2Message {
-    account: String,
-    action: String,
-    bucket: String,
-    object: R2Object,
-}
-
-#[event(queue)]
-async fn queue(message_batch: MessageBatch<R2Message>, env: Env, _ctx: Context) -> Result<()> {
-    let index = env.d1("INDEX")?;
-    let bucket = env.bucket("GABRIELSIMMERCOM_BUCKET")?;
-    let messages = message_batch.messages()?;
-    for message in messages {
-        let fname = message.into_body().object.key;
-        let dir: Vec<&str> = fname.split("/").collect();
-        if dir[0] != "posts" {
-            continue
-        }
-        console_log!("got {} to render and index", &fname);
-        let file = bucket.get(&fname).execute().await?;
-        if file.is_none() {
-            console_warn!("did not find a file in bucket for {}", fname)
-        }
-        let rendered = blog_post(fname.clone(), file.unwrap().body().unwrap().text().await?).unwrap();
-
-        let statement = index.prepare("INSERT INTO posts (slug, title, html, date) VALUES (?1, ?2, ?3, ?4) ON CONFLICT(slug) DO UPDATE SET html=?3;");
-        let query = statement.bind(&[
-            rendered.slug.into(), rendered.title.into(), rendered.html.into(), rendered.date.into()
-        ]).unwrap();
-        let _result = query.run().await?;
-    }
-
-    Ok(())
-}

queues/wrangler.toml

@@ -1,25 +0,0 @@
-name = "gabrielsimmercom-queues"
-main = "result/shim.mjs"
-compatibility_date = "2023-10-22"
-
-account_id = "7dc420732ea679a530aee304ea49a63c"
-workers_dev = true
-
-[limits]
-cpu_ms = 100
-
-[build]
-command = "cargo install -q worker-build && worker-build --release"
-
-[[r2_buckets]]
-binding = 'GABRIELSIMMERCOM_BUCKET'
-bucket_name = 'gabrielsimmer-com'
-preview_bucket_name = 'gabrielsimmer-com-dev'
-
-[[d1_databases]]
-binding = "INDEX"
-database_name = "gabrielsimmercom"
-database_id = "53acce7f-f529-4392-aa41-14084c445af0"
-
-[[queues.consumers]]
-queue = "gabrielsimmercom-indexing"

shared/Cargo.toml

@@ -1,23 +0,0 @@
-[package]
-name = "shared"
-version = "0.1.0"
-edition = "2021"
-
-[lib]
-crate-type = ["cdylib", "rlib"]
-
-[dependencies]
-maud = { version = "*" }
-serde = { version = "1.0.167", features = ["derive"] }
-frontmatter = "0.4.0"
-comrak = { version = "0.21.0", default-features = false }
-orgize = { git = "https://git.gmem.ca/arch/orgize.git", branch = "org-images", default-features = false }
-
-[profile.release]
-lto = true
-strip = true
-codegen-units = 1
-opt-level = 's'
-
-[package.metadata.wasm-pack.profile.release]
-wasm-opt = false

shared/src/lib.rs

@@ -1,69 +0,0 @@
-use maud::{html, Markup, Render};
-use orgize::Org;
-use serde::{Serialize, Deserialize};
-use std::{path::Path, ffi::OsStr};
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct PostMetadata {
-    pub slug: String,
-    pub title: String,
-    pub date: String,
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct PostContent {
-    pub slug: String,
-    pub title: String,
-    pub date: String,
-    pub html: String,
-}
-
-impl Render for PostMetadata {
-    fn render(&self) -> Markup {
-        html! {
-            li { (self.date) " - " a href=(format!("/blog/{}", self.slug)) { (self.title) } }
-        }
-    }
-}
-
-// Render the actual blog post as HTML.
-pub fn blog_post(fname: String, post: String) -> Result<PostContent, bool> {
-    // Search through /posts directory and find the post with either .md or .org extension
-    // If the post is not found, return 404
-
-    let ext = Path::new(&fname).extension().and_then(OsStr::to_str).unwrap();
-    let ext_name = format!(".{}", ext);
-    let slug = fname.replace("posts/", "").replace( &ext_name, "");
-    let mut html = "".to_owned();
-    let mut date = "".to_owned();
-    let mut title = "".to_owned();
-    if ext == "md" {
-        let (parsed, content) = frontmatter::parse_and_find_content(&post).unwrap();
-        let metadata = parsed.unwrap();
-        date = metadata["date"].as_str().unwrap().to_owned();
-        title = metadata["title"].as_str().unwrap().to_owned();
-        html = comrak::markdown_to_html(&content, &comrak::ComrakOptions::default());
-    } else if ext == "org" {
-        let mut writer = Vec::new();
-
-        let parsed = Org::parse(&post);
-        let keywords = parsed.keywords();
-        // Get date and title from keywords iterator
-
-        for keyword in keywords {
-            if keyword.key == "date" {
-                date = keyword.value.to_string();
-            } else if keyword.key == "title" {
-                title = keyword.value.to_string();
-            }
-        }
-        parsed.write_html(&mut writer).unwrap();
-        html = String::from_utf8(writer).unwrap();
-    }
-    Ok(PostContent {
-        slug,
-        title,
-        date,
-        html,
-    })
-}

src/cache/mod.rs

@@ -0,0 +1,56 @@
+use std::collections::HashMap;
+use std::sync::{Mutex, OnceLock};
+
+#[derive(Clone, Debug)]
+pub struct CachedItem {
+    pub content_type: String,
+    pub content: String,
+    pub cached: i64,
+}
+
+/// Determine whether we should actually use the cached item or not.
+fn should_use(_item: &CachedItem) -> bool {
+    // let current_time: i64 = SystemTime::now()
+    //     .duration_since(UNIX_EPOCH)
+    //     .expect("SystemTime before UNIX EPOCH!")
+    //     .as_secs()
+    //     .try_into()
+    //     .unwrap();
+
+    // current_time <= (item.cached + (2*60)) && item.content != ""
+    true
+}
+
+static CACHE: OnceLock<Mutex<HashMap<String, CachedItem>>> = OnceLock::new();
+
+pub async fn get(key: &String) -> Option<CachedItem> {
+    let data = CACHE
+        .get_or_init(|| Mutex::new(HashMap::new()))
+        .lock()
+        .unwrap();
+    match data.get(key) {
+        Some(c) => {
+            if should_use(&c) {
+                Some(c.clone())
+            } else {
+                //let _rm = rm(key.to_string()).await;
+                None
+            }
+        }
+        None => None,
+    }
+}
+
+// async fn rm(key: String) {
+//     let mut data = CACHE.get().unwrap().clone();
+//     data.remove(&key);
+//     let _ = CACHE.set(data);
+// }
+
+pub async fn set(key: String, item: CachedItem) {
+    let mut data = CACHE
+        .get_or_init(|| Mutex::new(HashMap::new()))
+        .lock()
+        .unwrap();
+    data.insert(key, item);
+}

src/main.rs

@@ -0,0 +1,546 @@
+// Include the `items` module, which is generated from items.proto.
+pub mod items {
+    include!(concat!(env!("OUT_DIR"), "/gabrielsimmerdotcom.gossip.rs"));
+}
+
+mod cache;
+mod posts;
+
+use axum::extract::Path;
+use axum::response::IntoResponse;
+use axum::{
+    body::Full,
+    extract::State,
+    http::StatusCode,
+    http::{Request, Response},
+    middleware::{self, Next},
+    routing::get,
+    Router,
+};
+use cache::CachedItem;
+use clap::Parser;
+use crossbeam::channel::{unbounded, Receiver, Sender};
+use file_format::{FileFormat, Kind};
+use hyper::body::Bytes;
+use maud::{html, Markup, PreEscaped, Render, DOCTYPE};
+use prost::Message;
+use rand::seq::SliceRandom;
+use rss::ChannelBuilder;
+use serde::Deserialize;
+use std::collections::HashMap;
+use std::net::UdpSocket;
+use std::time::{SystemTime, UNIX_EPOCH};
+use std::{io, thread};
+use time::{self, format_description, format_description::well_known::Rfc2822};
+use tower_http::services::ServeDir;
+
+#[derive(Debug)]
+struct Peer {
+    counter: i64,
+    health: items::Status,
+}
+
+#[derive(Parser)]
+struct Cli {
+    #[arg(short, long, default_value_t=("0.0.0.0:3000").to_string())]
+    bind: String,
+    #[arg(short, long)]
+    peers: Option<String>,
+    #[arg(long, default_value_t=("0.0.0.0:1337").to_string())]
+    gossip_bind: String,
+}
+
+#[derive(Clone, Debug)]
+struct AppState {
+    to_gossip: Sender<items::Payload>,
+}
+
+#[derive(Deserialize)]
+struct Project {
+    name: String,
+    description: String,
+    link: ProjectUrl,
+    languages: Vec<String>,
+}
+
+#[derive(Deserialize)]
+struct ProjectUrl {
+    display_text: String,
+    link: String,
+}
+
+#[derive(Deserialize)]
+struct ProjectConfig {
+    projects: Vec<Project>,
+    experiments: Vec<Project>,
+}
+
+impl Render for Project {
+    fn render(&self) -> Markup {
+        html! {
+            div .project {
+                h4 { ( self.name ) " " }
+                @for language in &self.languages {
+                    span .language .(language.to_lowercase()) { ( language ) }
+                }
+                p { ( self.description ) " " a href=(self.link.link) { (self.link.display_text) }}
+            }
+        }
+    }
+}
+
+#[tokio::main]
+async fn main() -> Result<(), ()> {
+    let args = Cli::parse();
+
+    // Create channels for sending messages and receiving results
+    let (s_gossip, r_gossip) = unbounded::<items::Payload>();
+    let (s_main, r_main) = unbounded::<items::Payload>();
+    let state = AppState {
+        to_gossip: s_main.clone(),
+    };
+
+    let app = Router::new()
+        .route("/", get(homepage))
+        .route("/rss", get(rss))
+        .route("/blog", get(list_blog_posts))
+        .route("/blog/:post", get(render_blog_post))
+        .route("/blog/:post/raw", get(raw_blog_post))
+        .nest_service("/assets", ServeDir::new("assets"))
+        .nest_service("/images", ServeDir::new("assets/images"))
+        .layer(middleware::from_fn_with_state(state.clone(), cached_page))
+        .with_state(state);
+
+    println!("Running webserver on {}", args.bind);
+    let webserver = axum::Server::bind(&args.bind.parse().unwrap()).serve(app.into_make_service());
+
+    if args.peers.is_some() {
+        tokio::spawn(webserver);
+        println!("starting gossip worker");
+        // Spawn a worker thread
+        let _gossip_server = thread::spawn(move || {
+            let binding = args.peers.unwrap();
+            let peer_list: Vec<&str> = binding.split(",").collect();
+            let _ = gossiper(
+                &args.gossip_bind,
+                peer_list,
+                s_main.clone(),
+                r_main,
+                s_gossip,
+            );
+        });
+        loop {
+            let message = r_gossip.recv().unwrap();
+            match &message.msg {
+                Some(items::payload::Msg::Cache(cache)) => {
+                    cache::set(
+                        cache.key.clone(),
+                        CachedItem {
+                            content_type: cache.content_type.clone(),
+                            content: cache.content.clone(),
+                            cached: cache.timestamp,
+                        },
+                    )
+                    .await;
+                }
+                _ => {}
+            }
+        }
+    } else {
+        let _ = webserver.await;
+    }
+
+    Ok(())
+}
+
+fn gossiper(
+    bind: &String,
+    peer_list: Vec<&str>,
+    own_sender: Sender<items::Payload>,
+    rx: Receiver<items::Payload>,
+    tx: Sender<items::Payload>,
+) -> io::Result<()> {
+    let mut peers: HashMap<String, Peer> = HashMap::new();
+    for peer in peer_list {
+        peers.insert(
+            peer.to_owned(),
+            Peer {
+                counter: 0,
+                health: items::Status::PeerAlive,
+            },
+        );
+    }
+    let mut buf = [0; 10240];
+    let socket = UdpSocket::bind(bind.as_str())?;
+    let r_socket = socket.try_clone().unwrap();
+    // When we come up, try to communicate we're alive to other peers.
+    let selected_peers: Vec<String> = match select_peers(&peers) {
+        Some(p) => {
+            println!("found peers");
+            p
+        }
+        None => {
+            println!("no peers, not gossiping");
+            vec![]
+        }
+    };
+    for peer in selected_peers {
+        let membership = items::Payload {
+            msg: Some(items::payload::Msg::Membership(items::Membership {
+                peer: bind.to_string(),
+                status: items::Status::PeerAlive as i32,
+                inc: 0,
+            })),
+            peer: bind.to_string(),
+        };
+        let response = Message::encode_to_vec(&membership);
+        let _result = match r_socket.send_to(&response, &peer) {
+            Ok(_) => "",
+            Err(_) => "",
+        };
+        let existing = peers.get(&peer).unwrap();
+        peers.insert(
+            peer,
+            Peer {
+                counter: existing.counter + 1,
+                health: items::Status::PeerAlive,
+            },
+        );
+    }
+    let bound = bind.clone();
+    thread::spawn(move || loop {
+        let (size, source) = socket.recv_from(&mut buf).expect("Failed to receive data");
+        let msg: Result<items::Payload, _> = Message::decode(&buf[..size]);
+
+        let _ = own_sender.send(msg.clone().unwrap()).unwrap();
+        if msg.unwrap().msg != Some(items::payload::Msg::Ack(items::Ack {})) {
+            let ack = items::Payload {
+                peer: bound.to_string(),
+                msg: Some(items::payload::Msg::Ack(items::Ack {})),
+            };
+            let res = Message::encode_to_vec(&ack);
+            socket
+                .send_to(&res, source)
+                .expect("Failed to send response");
+        }
+    });
+
+    // Handle messages that are passed to us from the main thread or the UDP socket.
+    loop {
+        let message = rx.recv().unwrap();
+        match &message.msg {
+            Some(items::payload::Msg::Cache(cache)) => {
+                println!("got new cache: {}", cache.content_type);
+                let _ = tx.send(message.clone()).unwrap();
+            }
+            Some(items::payload::Msg::Ack(_)) => {
+                println!("healthy ack");
+                let peer = message.peer;
+                let existing = peers.get(&peer).unwrap();
+                peers.insert(
+                    peer,
+                    Peer {
+                        counter: existing.counter + 1,
+                        health: items::Status::PeerAlive,
+                    },
+                );
+            }
+            Some(items::payload::Msg::Ping(_)) => todo!(),
+            Some(items::payload::Msg::Membership(membership)) => {
+                println!("membership update");
+                let peer = message.peer;
+                let existing = peers.get(&peer).unwrap();
+
+                peers.insert(
+                    peer,
+                    Peer {
+                        counter: existing.counter + 1,
+                        health: membership.status(),
+                    },
+                );
+            }
+            Some(items::payload::Msg::NewCache(new_cache)) => {
+                println!("instructed to send cache");
+                let current_time = SystemTime::now()
+                    .duration_since(UNIX_EPOCH)
+                    .expect("SystemTime before UNIX EPOCH!")
+                    .as_secs()
+                    .try_into()
+                    .unwrap();
+                let cache = items::Payload {
+                    peer: bind.to_owned(),
+                    msg: Some(items::payload::Msg::Cache(items::Cache {
+                        key: new_cache.key.clone(),
+                        content: new_cache.content.clone(),
+                        content_type: new_cache.content_type.clone(),
+                        timestamp: current_time,
+                    })),
+                };
+                let selected_peers: Vec<String> = match select_peers(&peers) {
+                    Some(p) => {
+                        println!("found peers");
+                        p
+                    }
+                    None => {
+                        println!("no peers, not gossiping");
+                        vec![]
+                    }
+                };
+                let payload = Message::encode_to_vec(&cache);
+                for peer in selected_peers {
+                    dbg!(&peer);
+                    let _result = match r_socket.send_to(&payload, &peer) {
+                        Ok(_r) => {
+                            // dbg!(r);
+                        }
+                        Err(e) => {
+                            dbg!(e);
+                        }
+                    };
+                    let p = peers.get_mut(&peer).unwrap();
+                    p.counter = p.counter + 1;
+                }
+            }
+            None => {}
+        }
+    }
+}
+
+/// select a third of the peers we know about to gossip to.
+fn select_peers(peers: &HashMap<String, Peer>) -> Option<Vec<String>> {
+    if peers.len() == 0 {
+        return None;
+    }
+    let healthy: Vec<String> = peers
+        .into_iter()
+        .filter(|(&ref k, &ref peer)| matches!(peer.health, items::Status::PeerAlive))
+        .map(|(&ref ip, &ref peer)| ip.to_owned())
+        .collect();
+    let mut len_rounded = healthy.len();
+    let rng = &mut rand::thread_rng();
+
+    if len_rounded > 3 {
+        let len: f64 = (healthy.len() / 3) as f64;
+        len_rounded = len.ceil() as usize;
+    }
+    // Select a random number of peers to gossip to.
+    Some(
+        healthy
+            .choose_multiple(rng, len_rounded)
+            .map(|f| f.to_owned())
+            .collect(),
+    )
+}
+
+async fn raw_blog_post(Path(post): Path<String>) -> Result<impl IntoResponse, StatusCode> {
+    let post = posts::blog_post(post);
+    if post.is_err() {
+        return Err(StatusCode::NOT_FOUND);
+    }
+    Ok(Response::builder()
+        .header("content-type", "text/plain")
+        .status(StatusCode::OK)
+        .body(Full::from(post.unwrap().content))
+        .unwrap())
+}
+
+async fn render_blog_post(Path(post): Path<String>) -> Result<impl IntoResponse, StatusCode> {
+    let post = posts::blog_post(post);
+    if post.is_err() {
+        return Err(StatusCode::NOT_FOUND);
+    }
+    let p = post.unwrap();
+    let html_maud = PreEscaped(p.html);
+
+    let html = html! {
+        (header(p.title.as_str()))
+            body {
+                main {
+                    h1 { (p.title) }
+                    p { (p.date) }
+                    (html_maud)
+                }
+            }
+    };
+    Ok(Response::builder()
+        .header("content-type", "text/html")
+        .status(StatusCode::OK)
+        .body(Full::from(html.into_string()))
+        .unwrap())
+}
+
+async fn rss() -> Result<impl IntoResponse, StatusCode> {
+    let posts = posts::get_posts();
+    let rss_posts: Vec<rss::Item> = posts.into_iter().map(|p| {
+		let date = format!("{} 00:00:00 +00:00:00", p.date);
+		let format = format_description::parse("[year]-[month]-[day] [hour]:[minute]:[second] [offset_hour sign:mandatory]:[offset_minute]:[offset_second]").unwrap();
+		let pub_date = match time::OffsetDateTime::parse(&date, &format).unwrap().format(&Rfc2822) {
+			Ok(r) => r,
+			Err(e) => { dbg!(e.to_string()); "".to_owned() },
+		};
+		rss::ItemBuilder::default()
+			.title(Some(p.name))
+			.link(Some(format!("https://gabrielsimmer.com/blog/{}", p.route)))
+			.pub_date(Some(pub_date))
+			.build()
+	}).collect();
+    let channel = ChannelBuilder::default()
+        .title("Gabriel Simmer's Blog".to_owned())
+        .link("https://gabrielsimmer.com/blog".to_owned())
+        .description("Gabriel Simmer's Blog Posts.".to_owned())
+        .items(rss_posts)
+        .build();
+
+    return Ok(Response::builder()
+        .header("content-type", "application/rss+xml")
+        .status(StatusCode::OK)
+        .body(Full::from(channel.to_string()))
+        .unwrap());
+}
+
+fn header(page_title: &str) -> Markup {
+    html! {
+        (DOCTYPE)
+            meta charset="utf-8";
+        meta name="viewport" content="width=device-width, initial-scale=1";
+        title { (page_title) }
+        link rel="stylesheet" href="/assets/styles.css";
+        link rel="alternate" type="application/rss+xml" title="Gabriel Simmer's Blog" href="/rss";
+        script async src="https://analytics.eu.umami.is/script.js" data-website-id="6cb8d5a5-703a-4b37-9e39-64c96a9a84c1" { };
+    }
+}
+
+async fn homepage() -> Markup {
+    let projects: ProjectConfig = serde_dhall::from_file("./dhall/projects.dhall")
+        .parse()
+        .unwrap();
+
+    html! {
+        (header("/"))
+            body {
+                main {
+                    h1 { "Gabriel Simmer" }
+                    h2 { "Infrastructure and DevOps" }
+                    p { a href = "/blog" { "Blog" } " " a href = "https://floofy.tech/@arch" rel = "me" { "Fediverse" } }
+                    h3 { "Projects" }
+                    @for project in projects.projects {
+                        (project)
+                    }
+                    h3 { "Experiments" }
+                    @for project in projects.experiments {
+                        (project)
+                    }
+                }
+            }
+    }
+}
+
+async fn list_blog_posts() -> Markup {
+    let posts = posts::get_posts();
+    html! {
+        (header("/blog"))
+            body {
+                main {
+                    h1 { "Blog" }
+                    a href = "/rss" { "RSS" }
+                    ul {
+                        @for post in posts {
+                            (post);
+                        }
+                    }
+                }
+            }
+    }
+}
+
+async fn cached_page<T>(
+    State(state): State<AppState>,
+    request: Request<T>,
+    next: Next<T>,
+) -> Response<Full<Bytes>> {
+    let path = request.uri().path().to_string();
+
+    let item = cache::get(&path).await;
+    if item.is_none() {
+        let res = next.run(request).await;
+        let (res_parts, res_body) = res.into_parts();
+        let bytes = match hyper::body::to_bytes(res_body).await {
+            Ok(bytes) => bytes,
+            Err(_err) => {
+                return Response::builder()
+                    .status(StatusCode::INTERNAL_SERVER_ERROR)
+                    .body(Full::from("error"))
+                    .unwrap()
+            }
+        };
+
+        let res = bytes.to_vec();
+        let contenttype = match res_parts.headers.get("content-type") {
+            Some(c) => c.to_str().unwrap(),
+            None => "text/plain",
+        };
+
+        if !res_parts.status.is_success() {
+            return Response::builder()
+                .header("content-type", contenttype)
+                .status(res_parts.status)
+                .body(Full::from(bytes))
+                .unwrap();
+        }
+
+        // Make sure we only cache text.
+        let format = FileFormat::from_bytes(&res);
+        if format.kind() != Kind::Text && format.kind() != Kind::Application {
+            return Response::builder()
+                .header("content-type", contenttype)
+                .header("cache", "not")
+                .status(StatusCode::OK)
+                .body(Full::from(bytes))
+                .unwrap();
+        }
+
+        let current_time = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .expect("SystemTime before UNIX EPOCH!")
+            .as_secs()
+            .try_into()
+            .unwrap();
+        let content = String::from_utf8(res).unwrap();
+        cache::set(
+            path.clone(),
+            CachedItem {
+                content_type: contenttype.to_owned(),
+                content: content.clone(),
+                cached: current_time,
+            },
+        )
+        .await;
+        let gossip = items::Payload {
+            peer: "".to_owned(),
+            msg: Some(items::payload::Msg::NewCache(items::NewCache {
+                key: path,
+                content,
+                content_type: contenttype.to_owned(),
+            })),
+        };
+        match state.to_gossip.send(gossip) {
+            Ok(_) => {}
+            Err(_) => {}
+        };
+
+        return Response::builder()
+            .header("content-type", contenttype)
+            .header("cache", "miss")
+            .status(StatusCode::OK)
+            .body(Full::from(bytes))
+            .unwrap();
+    } else {
+        let i = item.unwrap();
+        return Response::builder()
+            .header("content-type", &i.content_type)
+            .header("cache", format!("hit"))
+            .status(StatusCode::OK)
+            .body(Full::from(i.content))
+            .unwrap();
+    }
+}

src/posts.rs

@@ -0,0 +1,126 @@
+use std::{
+    fs::{self, File},
+    io::Read,
+};
+
+use maud::{html, Markup, Render};
+use orgize::Org;
+
+pub struct PostMetadata {
+    pub name: String,
+    pub route: String,
+    pub date: String,
+}
+
+pub struct PostContent {
+    pub title: String,
+    pub date: String,
+    pub content: String,
+    pub html: String,
+}
+
+impl Render for PostMetadata {
+    fn render(&self) -> Markup {
+        html! {
+            li { (self.date) " - " a href=(format!("/blog/{}", self.route)) { (self.name) } }
+        }
+    }
+}
+
+pub fn get_posts() -> Vec<PostMetadata> {
+    let mut posts: Vec<PostMetadata> = Vec::new();
+    for entry in fs::read_dir("./posts").unwrap() {
+        let entry = entry.unwrap();
+        let path = entry.path();
+        let filename = path.file_name().unwrap().to_str().unwrap();
+        let ext = path.extension().unwrap().to_str().unwrap();
+
+        // strip extension
+        let fname = filename.replace(&format!(".{}", ext), "");
+        if ext == "md" || ext == "org" {
+            // We'll have the date at the beginning of the file
+            let mut content = File::open(&path).unwrap();
+            let mut buffer = [0; 100];
+            content.read(&mut buffer).unwrap();
+            // Match date data of `date: YYYY-MM-DD` in the first 100 bytes
+            let metadata = String::from_utf8_lossy(&buffer);
+            let metadata_lines = metadata.split("\n").collect::<Vec<&str>>();
+            // dbg!(&metadata);
+            // Split by --- and get the second element
+            let date = metadata_lines
+                .iter()
+                .find(|&x| x.contains("date:"))
+                .unwrap_or(&"")
+                .split(":")
+                .collect::<Vec<&str>>()[1];
+            let title = metadata_lines
+                .iter()
+                .find(|&x| x.contains("title:"))
+                .unwrap_or(&"")
+                .split(":")
+                .collect::<Vec<&str>>()[1]
+                .trim();
+            let date = date.trim();
+
+            posts.push(PostMetadata {
+                name: title.to_owned(),
+                route: fname,
+                date: date.to_owned(),
+            });
+        }
+    }
+    posts.sort_by(|a, b| b.date.cmp(&a.date));
+    posts
+}
+
+// Render the actual blog post as HTML.
+pub fn blog_post(post: String) -> Result<PostContent, bool> {
+    // Search through /posts directory and find the post with either .md or .org extension
+    // If the post is not found, return 404
+    for entry in fs::read_dir("./posts").unwrap() {
+        let entry = entry.unwrap();
+        let path = entry.path();
+        let filename = path.file_name().unwrap().to_str().unwrap();
+        let ext = path.extension().unwrap().to_str().unwrap();
+        // strip extension
+        let fname = filename.replace(&format!(".{}", ext), "");
+        if fname == post && (ext == "md" || ext == "org") {
+            let content = fs::read_to_string(&path).unwrap();
+
+            let mut html = "".to_owned();
+            let mut date = "".to_owned();
+            let mut title = "".to_owned();
+
+            if ext == "md" {
+                let (parsed, content) = frontmatter::parse_and_find_content(&content).unwrap();
+                let metadata = parsed.unwrap();
+                date = metadata["date"].as_str().unwrap().to_owned();
+                title = metadata["title"].as_str().unwrap().to_owned();
+                html = comrak::markdown_to_html(&content, &comrak::ComrakOptions::default());
+            } else if ext == "org" {
+                let mut writer = Vec::new();
+                let parsed = Org::parse(&content);
+                let keywords = parsed.keywords();
+                // Get date and title from keywords iterator
+
+                for keyword in keywords {
+                    if keyword.key == "date" {
+                        date = keyword.value.to_string();
+                    } else if keyword.key == "title" {
+                        title = keyword.value.to_string();
+                    }
+                }
+                parsed.write_html(&mut writer).unwrap();
+                html = String::from_utf8(writer).unwrap();
+            }
+            return Ok(PostContent {
+                title,
+                date,
+                content,
+                html,
+            });
+        }
+    }
+
+    return Err(false);
+}

web/Cargo.toml

@@ -1,22 +0,0 @@
-[package]
-name = "web"
-version = "0.1.0"
-edition = "2021"
-
-[lib]
-crate-type = ["cdylib", "rlib"]
-
-[dependencies]
-shared = { path = "../shared" }
-worker = { version = "0.1.0", features = ["d1"] }
-maud = { version = "*" }
-serde_dhall = { version = "0.12.1", default-features = false }
-serde = { version = "1.0.167", features = ["derive"] }
-include_dir = "0.7.3"
-rss = { version = "2.0.6" }
-time = { version = "0.3.31", features = ["parsing", "formatting", "macros", "wasm-bindgen"] }
-
-[profile.release]
-lto = true
-strip = true
-codegen-units = 1

web/src/lib.rs

@@ -1,229 +0,0 @@
-use shared::*;
-use worker::*;
-use maud::{html, Markup, Render, DOCTYPE};
-use serde::Deserialize;
-use include_dir::{include_dir, Dir};
-use time::{self, macros::format_description, format_description::well_known::Rfc2822};
-
-#[derive(Deserialize, Debug)]
-struct Project {
-    name: String,
-    description: String,
-    link: ProjectUrl,
-    languages: Vec<String>,
-}
-
-#[derive(Deserialize, Debug)]
-struct ProjectUrl {
-    display_text: String,
-    link: String,
-}
-
-#[derive(Deserialize, Debug)]
-struct ProjectConfig {
-    projects: Vec<Project>,
-    experiments: Vec<Project>,
-}
-
-static DHALL_DIR: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/src/dhall");
-
-impl Render for Project {
-    fn render(&self) -> Markup {
-        html! {
-            div .project {
-                h4 { ( self.name ) " " }
-                @for language in &self.languages {
-                    span .language .(language.to_lowercase()) { ( language ) }
-                }
-                p { ( self.description ) " " a href=(self.link.link) { (self.link.display_text) }}
-            }
-        }
-    }
-}
-
-#[event(fetch)]
-async fn main(req: Request, env: Env, _ctx: Context) -> Result<Response> {
-    let router = Router::new();
-    router
-        .get_async("/", |req, _ctx| async move {
-            let cache = Cache::default();
-            let key = req.url()?.to_string();
-            if let Some(mut res) = cache.get(&key, false).await? {
-                let headers = res.headers_mut();
-                let _ = headers.set("cf-cache", "hit");
-                return Ok(res)
-            }
-
-            let mut res = Response::from_html(homepage().await.into_string())?;
-            let headers = res.headers_mut();
-            let _ = headers.set("Cache-Control", "public, max-age=14400");
-            cache.put(key, res.cloned()?).await?;
-            Ok(res)
-        })
-        .get_async("/assets/styles.css", |req, _ctx| async move {
-            let cache = Cache::default();
-            let key = req.url()?.to_string();
-            if let Some(mut res) = cache.get(&key, false).await? {
-                let headers = res.headers_mut();
-                let _ = headers.set("cf-cache", "hit");
-                return Ok(res)
-            }
-            
-            let css = include_str!("styles.css");
-            let mut res = Response::ok(css)?;
-            let headers = res.headers_mut();
-            let _ = headers.set("Content-Type", "text/css");
-            let _ = headers.set("Cache-Control", "public, max-age=14400");
-            cache.put(key, res.cloned()?).await?;
-            Ok(res)
-        })
-        .get_async("/blog", |req, ctx| async move {
-            let cache = Cache::default();
-            let key = req.url()?.to_string();
-            if let Some(mut res) = cache.get(&key, false).await? {
-                let headers = res.headers_mut();
-                let _ = headers.set("cf-cache", "hit");
-                return Ok(res);
-            }
-            
-            let index = ctx.env.d1("INDEX")?;
-            let statement = index.prepare("SELECT * FROM posts ORDER BY date DESC");
-            let posts: Vec<PostMetadata> = statement.all().await?.results()?;
-
-            let html = html! {
-                (header("/blog"))
-                    body {
-                        main {
-                            h1 { "Blog" }
-                            a href = "/rss" { "RSS" }
-                            ul {
-                                @for post in posts {
-                                    (post);
-                                }
-                            }
-                        }
-                    }
-            };
-            let mut res = Response::from_html(html.into_string())?;
-            let headers = res.headers_mut();
-            let _ = headers.set("Cache-Control", "public, max-age=14400");
-            cache.put(key, res.cloned()?).await?;
-            return Ok(res);
-        })
-        .get_async("/blog/:slug", |req, ctx| async move {
-            let cache = Cache::default();
-            let key = req.url()?.to_string();
-            if let Some(mut res) = cache.get(&key, false).await? {
-                let headers = res.headers_mut();
-                let _ = headers.set("cf-cache", "hit");
-                return Ok(res)
-            }
-            
-            let index = ctx.env.d1("INDEX")?;
-            let slug = ctx.param("slug").unwrap();
-            let statement = index.prepare("SELECT * FROM posts WHERE slug=?1");
-            let query = statement.bind(&[slug.into()])?;
-            let post = query.first::<shared::PostContent>(None).await?;
-
-            let p = post.unwrap();
-            let html_maud = maud::PreEscaped(p.html);
-
-            let html = html! {
-                (header(p.title.as_str()))
-                    body {
-                        main {
-                            h1 { (p.title) }
-                            p { (p.date) }
-                            (html_maud)
-                        }
-                    }
-            };
-
-            let mut res = Response::from_html(html.into_string())?;
-            let headers = res.headers_mut();
-            let _ = headers.set("Cache-Control", "public, max-age=14400");
-            cache.put(key, res.cloned()?).await?;
-            return Ok(res);            
-        })
-        .get_async("/rss", |req, ctx| async move {
-            let cache = Cache::default();
-            let key = req.url()?.to_string();
-            if let Some(mut res) = cache.get(&key, false).await? {
-                let headers = res.headers_mut();
-                let _ = headers.set("cf-cache", "hit");
-                return Ok(res)
-            }
-
-            let index = ctx.env.d1("INDEX")?;
-            let statement = index.prepare("SELECT * FROM posts ORDER BY date DESC");
-            let posts: Vec<PostMetadata> = statement.all().await?.results()?;
-            let rss_posts: Vec<rss::Item> = posts.into_iter().map(|p| {
-		        let date = format!("{} 00:00:00 +00:00:00", p.date);
-		        let format = format_description!("[year]-[month]-[day] [hour]:[minute]:[second] [offset_hour sign:mandatory]:[offset_minute]:[offset_second]");
-		        let parsed_date = match time::OffsetDateTime::parse(&date, &format) {
-			        Ok(r) => r,
-			        Err(_e) => { time::OffsetDateTime::now_utc() },
-		        };
-                let pub_date = match parsed_date.format(&Rfc2822) {
-                  	Ok(r) => r,
-			        Err(_e) => { "".to_owned() },
-                };
-		        rss::ItemBuilder::default()
-			        .title(Some(p.title))
-			        .link(Some(format!("https://gabrielsimmer.com/blog/{}", p.slug)))
-			        .pub_date(Some(pub_date))
-			        .build()
-	        }).collect();
-            let channel = rss::ChannelBuilder::default()
-                .title("Gabriel Simmer's Blog".to_owned())
-                .link("https://gabrielsimmer.com/blog".to_owned())
-                .description("Gabriel Simmer's Blog Posts.".to_owned())
-                .items(rss_posts)
-                .build();
-
-            let mut res = Response::ok(channel.to_string())?;
-            let headers = res.headers_mut();
-            let _ = headers.set("Cache-Control", "public, max-age=14400");
-            let _ = headers.set("Content-Type", "application/rss+xml");
-
-            cache.put(key, res.cloned()?).await?;
-            Ok(res)
-        })
-        .run(req, env).await
-}
-
-async fn homepage() -> Markup {
-    let projects: ProjectConfig = serde_dhall::from_str(DHALL_DIR.get_file("projects.dhall").unwrap().contents_utf8().unwrap())
-        .parse()
-        .unwrap();
-
-    html! {
-        (header("/"))
-            body {
-                main {
-                    h1 { "Gabriel Simmer" }
-                    h2 { "Infrastructure and DevOps" }
-                    p { a href = "/blog" { "Blog" } " " a href = "https://floofy.tech/@arch" rel = "me" { "Fediverse" } }
-                    h3 { "Projects" }
-                    @for project in projects.projects {
-                        (project)
-                    }
-                    h3 { "Experiments" }
-                    @for project in projects.experiments {
-                        (project)
-                    }
-                }
-            }
-    }
-}
-
-fn header(page_title: &str) -> Markup {
-    html! {
-        (DOCTYPE)
-            meta charset="utf-8";
-        meta name="viewport" content="width=device-width, initial-scale=1";
-        title { (page_title) }
-        link rel="stylesheet" href="/assets/styles.css";
-        link rel="alternate" type="application/rss+xml" title="Gabriel Simmer's Blog" href="/rss";
-    }
-}

web/src/schema.sql

@@ -1,2 +0,0 @@
-DROP TABLE IF EXISTS posts;
-CREATE TABLE IF NOT EXISTS posts (slug TEXT PRIMARY KEY, title TEXT, html TEXT, date TEXT);

web/wrangler.toml

@@ -1,22 +0,0 @@
-name = "gabrielsimmercom-workers"
-main = "build/worker/shim.mjs"
-compatibility_date = "2023-10-22"
-
-account_id = "7dc420732ea679a530aee304ea49a63c"
-workers_dev = true
-
-[limits]
-cpu_ms = 100
-
-[build]
-command = "cargo install -q worker-build && worker-build --release"
-
-[[r2_buckets]]
-binding = 'GABRIELSIMMERCOM_BUCKET' # <~ valid JavaScript variable name
-bucket_name = 'gabrielsimmer-com'
-preview_bucket_name = 'gabrielsimmer-com-dev'
-
-[[d1_databases]]
-binding = "INDEX"
-database_name = "gabrielsimmercom"
-database_id = "53acce7f-f529-4392-aa41-14084c445af0"