AWS VPC with IPAM and a tailscale subnet router with optional fck-nat for private subnets for cost savings on SDLC enviornments

networking.ts

import * as aws from "@pulumi/aws";
import * as awsx from "@pulumi/awsx";
import * as pulumi from "@pulumi/pulumi";
import * as fckNat from "@pulumi/fck-nat";
import * as tailscale from "@pulumi/tailscale";

// These pools are managed by IPAM
// see: https://quatt.slite.com/app/docs/3yfj20n1t-LZOd#7869cd30
const ipamPools = {
  prod: {
    eucentral1: "10.6.0.0/15",
    euwest1: "10.2.0.0/15",
  },
  sdlc: {
    eucentral1: "10.4.0.0/15",
    euwest1: "10.0.0.0/15",
  },
};

type SetupVPCArgs = {
  // The IPAM pool ID to use for this VPC. A CIDR range of 256 ip's will be allocated from this pool
  ipamId: string;
  // this is needed to correctly tag the nat instance created by the fck-nat module
  serviceName: string;
  // With this enabled the vpc will use the fck-nat gateway instead of the AWS managed
  // NAT gateway for outgoing trafic. fck-nat is ALLOT cheaper then the AWS managed NAT gateway
  // and should be enabled on all non prod environments and prod where possible
  // when switching on a live environment you are better off destroying the vpc and
  // recreating it as pulumi up will fail due to the unique constraints
  useFckNatGateway: boolean;
  // default is t4g.micro which is usually fine for most use cases
  natInstanceType?: string;
};

// setupNetworking will deploy a VPC with quatt specific architecture
// we use the recommended 3 public and private subnets with a single NAT gateway
// On top of this we deploy a tailscale subnet router that can be used for both incomming
// and outgoing trafic and a few aws specific network endpoints to save costs
// SDLC vpc's are recomended to use the fck-nat gateway instead of the AWS managed NAT gateway
// to save costs. Prod vpc's can also use this if the trafic can survive short interruptions
export const setupNetworking = (
  prefix: string,
  args: SetupVPCArgs,
  opts?: pulumi.ComponentResourceOptions,
) => {
  const cfg = new pulumi.Config();
  const env = pulumi.getStack().includes("prod") ? "prod" : "sdlc";
  const project = pulumi.getProject();
  const stack = pulumi.getStack();

  const vpcx = new awsx.ec2.Vpc(
    `${prefix}-vpcx`,
    {
      ipv4IpamPoolId: args.ipamId,
      ipv4NetmaskLength: 24,
      subnetStrategy: "Auto",
      enableDnsHostnames: true,
      enableDnsSupport: true,
      natGateways: args.useFckNatGateway
        ? { strategy: "None" }
        : { strategy: "Single" },
    },
    opts,
  );

  const region = aws.getRegion({}, opts).then((r) => r.name);

  const s3Endpoint = new aws.ec2.VpcEndpoint(
    `${prefix}-s3-endpoint`,
    {
      vpcId: vpcx.vpcId,
      serviceName: pulumi.interpolate`com.amazonaws.${region}.s3`,
      routeTableIds: vpcx.routeTables.apply((x) => x.map((rt) => rt.id)),
      tags: {
        Name: `${prefix}-s3-endpoint`,
      },
    },
    opts,
  );

  // Get route table outputs for each private subnet
  // awsx doesn't expose this directly so we will need to
  // query aws for it
  const privateRouteTables = aws.ec2.getRouteTablesOutput(
    {
      filters: [
        {
          name: "association.subnet-id",
          values: vpcx.privateSubnetIds,
        },
      ],
    },
    opts,
  );

  const tsKey = new tailscale.TailnetKey(
    `${prefix}-tailscale-key`,
    {
      reusable: true,
      ephemeral: true,
      preauthorized: true,
      expiry: 0,
      description: `pulumi key for ${project} ${stack}`,
      // To dynamicly decide if we are deploying into the sdlc or production network we check for the word
      // `prod` in the pulumi stack name. This is not 100% accurate but it should cover all naming conventions
      // within our company
      tags: env === "prod" ? ["tag:prod"] : ["tag:sdlc"],
    },
    {
      provider: new tailscale.Provider(`${prefix}-tailscale-provider`, {
        oauthClientId: cfg.require("tsOauthClientId"),
        oauthClientSecret: cfg.requireSecret("tsOauthClientSecret"),
        scopes: ["auth_keys"],
      }),
      ignoreChanges: ["expiry"], // Workaround for the tailscale provider trying to redeploy a expire 0 key every up
    },
  );

  const nat = new fckNat.Module(`${prefix}-nat`, {
    name: `${prefix}-nat`,
    subnet_id: vpcx.publicSubnetIds.apply((ids) => ids[0]),
    vpc_id: vpcx.vpcId,
    update_route_tables: args.useFckNatGateway,
    instance_type: args.natInstanceType || "t4g.micro",
    route_tables_ids: privateRouteTables.apply((rts) =>
      rts.ids.reduce((acc, val, i) => {
        return { ...acc, [i]: val };
      }, {}),
    ),
    // we need to specifically set the tags here as it's difficult
    // to use the opts.Provider in the args for this specific terraform
    // module
    tags: {
      Provider: "Pulumi",
      Project: project,
      Service: args.serviceName,
      stack: stack,
    },
    cloud_init_parts: [
      {
        content_type: "text/x-shellscript",
        content: pulumi.interpolate`#!/bin/bash
dnf install -y dnf-utils
dnf config-manager --add-repo https://pkgs.tailscale.com/stable/amazon-linux/2/tailscale.repo
dnf install -y tailscale

systemctl enable --now tailscaled
echo "Enabling and starting tailscaled service..."
sleep 5
tailscale up \
  --advertise-routes=${vpcx.vpc.cidrBlock} \
  --advertise-tags=${env === "prod" ? ["tag:prod"] : ["tag:sdlc"]} \
  --hostname=${project}-${stack}-tailscale-subnet-router \
  --authkey=${tsKey.key}

# Enable IP forwarding for Tailscale traffic
echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.conf
echo 'net.ipv6.conf.all.forwarding = 1' >> /etc/sysctl.conf
sysctl -p

# Wait for Tailscale to be fully up and add kernel routing for Tailscale networks
sleep 10
while ! ip addr show tailscale0 >/dev/null 2>&1; do
  echo "Waiting for Tailscale interface..."
  sleep 5
done

# Create systemd service for persistent Tailscale routing
cat > /etc/systemd/system/tailscale-routing.service << 'EOF'
[Unit]
Description=Tailscale routing configuration for NAT gateway
After=tailscaled.service
Wants=tailscaled.service

[Service]
Type=oneshot
RemainAfterExit=yes
# tailscale devices
ExecStart=/bin/bash -c 'ip route add 100.64.0.0/10 dev tailscale0 2>/dev/null || true'
ExecStart=/bin/bash -c 'iptables -t nat -A POSTROUTING -s 10.0.0.0/8 -d 100.64.0.0/10 -j MASQUERADE'

# eucentral1 vpcs
ExecStart=/bin/bash -c 'ip route add ${ipamPools[env].eucentral1} dev tailscale0 2>/dev/null || true'
ExecStart=/bin/bash -c 'iptables -t nat -A POSTROUTING -s 10.0.0.0/8 -d ${ipamPools[env].eucentral1} -j MASQUERADE'

# euwest1 vpcs
ExecStart=/bin/bash -c 'ip route add ${ipamPools[env].euwest1} dev tailscale0 2>/dev/null || true'
ExecStart=/bin/bash -c 'iptables -t nat -A POSTROUTING -s 10.0.0.0/8 -d ${ipamPools[env].euwest1} -j MASQUERADE'

[Install]
WantedBy=multi-user.target
EOF

# Enable and start the service
systemctl enable tailscale-routing.service
systemctl start tailscale-routing.service
        `,
      },
    ],
  });

  // Add routes for Tailscale IP ranges to private subnet route tables
  // routes cant be double so we will need to delete them
  // before replacing to make sure there are no arrors during apply
  const tailscaleRoutes = privateRouteTables.apply((rts) =>
    rts.ids.map((routeTableId, index) => {
      return [
        new aws.ec2.Route(
          `${prefix}-tailscale-devices-route-${index}`,
          {
            routeTableId: routeTableId,
            destinationCidrBlock: "100.64.0.0/10", // Tailscale IP range
            networkInterfaceId: nat.eni_id,
          },
          opts,
        ),
        new aws.ec2.Route(
          `${prefix}-tailscale-eucentral1-route-${index}`,
          {
            routeTableId: routeTableId,
            destinationCidrBlock: ipamPools[env].eucentral1, // Tailscale IP range
            networkInterfaceId: nat.eni_id,
          },
          opts,
        ),
        new aws.ec2.Route(
          `${prefix}-tailscale-euwest1-route-${index}`,
          {
            routeTableId: routeTableId,
            destinationCidrBlock: ipamPools[env].euwest1, // Tailscale IP range
            networkInterfaceId: nat.eni_id,
          },
          opts,
        ),
      ];
    }),
  );

  return {
    vpcx,
    s3Endpoint,
    privateRouteTables,
    nat,
    tsKey,
    tailscaleRoutes,
  };
};