Create weekly link archive for Hugo blog

add_links.py

#!/usr/bin/env python3
"""
LinkWarden Weekly Links Generator

This script fetches links from LinkWarden API with a specific tag,
filters them to only include links from the last week,
and creates a markdown file for Hugo static site.
"""

import requests
import json
import os
import glob
from datetime import datetime, timedelta
from typing import List, Dict, Any
import argparse


class LinkWardenClient:
    def __init__(self, protocol: str, host: str, access_token: str):
        self.base_url = f"{protocol}://{host}/api/v1"
        self.headers = {
            "Content-Type": "application/json",
            "Authorization": f"Bearer {access_token}"
        }
    
    def get_links_by_tag(self, tag_id: int) -> List[Dict[str, Any]]:
        """Fetch links from LinkWarden API by tag ID"""
        url = f"{self.base_url}/links?tagId={tag_id}"
        
        try:
            response = requests.get(url, headers=self.headers)
            response.raise_for_status()
            
            data = response.json()
            return data.get('response', [])
        
        except requests.exceptions.RequestException as e:
            print(f"Error fetching links: {e}")
            return []
        except json.JSONDecodeError as e:
            print(f"Error parsing JSON response: {e}")
            return []


class LinkFilter:
    @staticmethod
    def filter_last_week(links: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
        """Filter links to only include those from the last week"""
        one_week_ago = datetime.now() - timedelta(days=7)
        filtered_links = []
        
        for link in links:
            try:
                # Parse the createdAt timestamp
                created_at = datetime.fromisoformat(link['createdAt'].replace('Z', '+00:00'))
                # Convert to local timezone for comparison
                created_at_local = created_at.replace(tzinfo=None)
                
                if created_at_local >= one_week_ago:
                    filtered_links.append(link)
            except (ValueError, KeyError) as e:
                print(f"Error parsing date for link {link.get('name', 'Unknown')}: {e}")
                continue
        
        return filtered_links


class MarkdownGenerator:
    def __init__(self, content_dir: str):
        self.content_dir = content_dir
        self.links_dir = os.path.join(content_dir, "links")
    
    def ensure_links_directory(self, year: str = None):
        """Create the links directory and year subdirectory if they don't exist"""
        if not os.path.exists(self.links_dir):
            os.makedirs(self.links_dir)
            print(f"Created directory: {self.links_dir}")
        
        if year:
            year_dir = os.path.join(self.links_dir, year)
            if not os.path.exists(year_dir):
                os.makedirs(year_dir)
                print(f"Created year directory: {year_dir}")
            return year_dir
        
        return self.links_dir
    
    def generate_markdown_file(self, links: List[Dict[str, Any]]) -> str:
        """Generate markdown file with the filtered links"""
        current_date = datetime.now()
        year = current_date.strftime('%Y')
        
        # Ensure year directory exists
        year_dir = self.ensure_links_directory(year)
        
        # Generate filename with month and day only (year is in the folder)
        filename = f"{current_date.strftime('%m%d')}.md"
        filepath = os.path.join(year_dir, filename)
        
        # Generate title with formatted date
        title = f"Interesting links from {current_date.strftime('%d-%m-%Y')}"
        
        # Generate markdown content
        content = self._generate_markdown_content(title, links, current_date)
        
        # Write to file
        try:
            with open(filepath, 'w', encoding='utf-8') as f:
                f.write(content)
            print(f"Created markdown file: {filepath}")
            print(f"Added {len(links)} links to the file")
            
            # Update the links overview file
            self.update_overview_file()
            
            return filepath
        except IOError as e:
            print(f"Error writing to file {filepath}: {e}")
            return ""
    
    def update_overview_file(self):
        """Create or update the link-overview.md file with references to all weekly link files organized by year"""
        links_file_path = os.path.join(self.content_dir, "link-overview.md")
        
        # Get all weekly link files from all year directories
        weekly_files = []
        
        # Look for year directories
        for item in os.listdir(self.links_dir):
            year_path = os.path.join(self.links_dir, item)
            if os.path.isdir(year_path) and item.isdigit() and len(item) == 4:  # Year directory
                # Get all MMDD.md files (new format) and YYYYMMDD.md files (legacy format) in this year directory
                new_pattern = os.path.join(year_path, "[0-9][0-9][0-9][0-9].md")
                legacy_pattern = os.path.join(year_path, "[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9].md")
                year_files = glob.glob(new_pattern) + glob.glob(legacy_pattern)
                weekly_files.extend([(f, item) for f in year_files])  # Store file path and year
        
        # Also check for any legacy files in the root links directory (YYYYMMDD.md format)
        pattern = os.path.join(self.links_dir, "[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9].md")
        legacy_files = glob.glob(pattern)
        weekly_files.extend([(f, None) for f in legacy_files])  # No year folder for legacy files
        
        # Sort files in descending order (newest first)
        weekly_files.sort(key=lambda x: x[0], reverse=True)
        
        # Generate overview content
        overview_content = self._generate_overview_content(weekly_files)
        
        # Write overview file
        try:
            with open(links_file_path, 'w', encoding='utf-8') as f:
                f.write(overview_content)
            print(f"Updated links file: {links_file_path}")
        except IOError as e:
            print(f"Error writing links file {links_file_path}: {e}")
    
    def _generate_overview_content(self, weekly_files: List[tuple]) -> str:
        """Generate the content for the overview.md file organized by year"""
        current_date = datetime.now()
        # Subtract 2 hours and round down to full hour
        adjusted_date = current_date - timedelta(hours=2)
        rounded_date = adjusted_date.replace(minute=0, second=0, microsecond=0)
        
        content = f"""---
title: "Weekly Links Archive"
date: {rounded_date.strftime('%Y-%m-%dT%H:%M:%S%z')}
draft: false
tags: ["links", "archive"]
type: "page"
---

## Weekly Links Archive

This page contains an archive of all my weekly link collections, automatically curated from my LinkWarden bookmarks.

"""
        
        if not weekly_files:
            content += "No weekly link files found yet.\n"
        else:
            # Group files by year
            files_by_year = {}
            for file_path, year_folder in weekly_files:
                filename = os.path.basename(file_path)
                # Extract date from filename
                date_str = filename.replace('.md', '')
                
                try:
                    # Handle different filename formats
                    if len(date_str) == 4:  # MMDD format (new)
                        # Year comes from the folder name
                        if year_folder:
                            full_date_str = f"{year_folder}{date_str}"
                            file_date = datetime.strptime(full_date_str, '%Y%m%d')
                            year = year_folder
                        else:
                            continue  # Skip if no year folder info
                    elif len(date_str) == 8:  # YYYYMMDD format (legacy)
                        file_date = datetime.strptime(date_str, '%Y%m%d')
                        year = file_date.strftime('%Y')
                    else:
                        continue  # Skip invalid formats
                    
                    if year not in files_by_year:
                        files_by_year[year] = []
                    files_by_year[year].append((file_path, file_date, date_str, year_folder))
                except ValueError:
                    # If date parsing fails, skip this file
                    continue
            
            # Sort years in descending order (newest first)
            for year in sorted(files_by_year.keys(), reverse=True):
                content += f"### {year}\n\n"
                
                # Sort files within year in descending order (newest first)
                year_files = sorted(files_by_year[year], key=lambda x: x[1], reverse=True)
                
                for file_path, file_date, date_str, year_folder in year_files:
                    formatted_date = file_date.strftime('%d-%m-%Y')
                    
                    # Create link to the weekly file using Hugo URL format
                    link_title = f"Interesting links from {formatted_date}"
                    
                    # Determine URL based on file location and format
                    if year_folder:  # File is in year subdirectory
                        if len(date_str) == 4:  # MMDD format
                            hugo_url = f"/links/{year}/{date_str}/"
                        else:  # YYYYMMDD format (legacy in year folder)
                            hugo_url = f"/links/{year}/{date_str}/"
                    else:  # Legacy file in root location
                        hugo_url = f"/links/{date_str}/"
                    
                    content += f"- [{link_title}]({hugo_url})\n"
                
                content += "\n"
        
        content += f"""---

### About

These weekly link collections are automatically generated every week from my personal bookmarks using LinkWarden. Each collection contains interesting articles, tools, and resources I discovered during that week.

*Last updated: {current_date.strftime('%d-%m-%Y at %H:%M')}*
"""
        
        return content
    
    def _generate_markdown_content(self, title: str, links: List[Dict[str, Any]], date: datetime) -> str:
        """Generate the actual markdown content"""
        # Subtract 2 hours and round down to full hour
        adjusted_date = date - timedelta(hours=2)
        rounded_date = adjusted_date.replace(minute=0, second=0, microsecond=0)
        
        content = f"""---
title: "{title}"
date: {rounded_date.strftime('%Y-%m-%dT%H:%M:%S%z')}
draft: false
tags: ["links", "weekly"]
---

## {title}

Here are some interesting links I discovered this week:

"""
        
        if not links:
            content += "No new links were found for this week.\n"
        else:
            for link in links:
                name = link.get('name', 'Untitled')
                url = link.get('url', '#')
                # Escape any special markdown characters in the name
                name_escaped = name.replace('[', '\\[').replace(']', '\\]')
                content += f"- [{name_escaped}]({url})\n"
        
        content += "\n---\n\n*This list was automatically generated from my LinkWarden bookmarks.*\n"
        
        return content


def main():
    parser = argparse.ArgumentParser(description='Generate weekly links markdown file from LinkWarden')
    parser.add_argument('--protocol', default='http', help='Protocol (http/https)')
    parser.add_argument('--host', help='LinkWarden host')
    parser.add_argument('--tag-id', type=int, default=5, help='Tag ID to query')
    parser.add_argument('--access-token', help='LinkWarden access token')
    parser.add_argument('--content-dir', default='../../content', help='Path to Hugo content directory')
    parser.add_argument('--update-overview-only', action='store_true', help='Only update the link-overview.md file without fetching new links')
    
    args = parser.parse_args()
    
    # Initialize markdown generator
    markdown_generator = MarkdownGenerator(args.content_dir)
    
    # If only updating overview, do that and exit
    if args.update_overview_only:
        print("Updating link-overview.md file only...")
        markdown_generator.update_overview_file()
        return
    
    # Require access token for API calls
    if not args.access_token:
        print("Error: --access-token is required when not using --update-overview-only")
        return
    
    # Initialize components
    client = LinkWardenClient(args.protocol, args.host, args.access_token)
    filter_service = LinkFilter()
    
    print(f"Fetching links with tag ID {args.tag_id} from {args.protocol}://{args.host}")
    
    # Fetch all links with the specified tag
    all_links = client.get_links_by_tag(args.tag_id)
    print(f"Found {len(all_links)} total links")
    
    # Filter to last week only
    recent_links = filter_service.filter_last_week(all_links)
    print(f"Found {len(recent_links)} links from the last week")
    
    # Generate markdown file
    if recent_links or True:  # Generate file even if no links (for consistency)
        markdown_generator.generate_markdown_file(recent_links)
    else:
        print("No recent links found, skipping file generation")


if __name__ == "__main__":
    main()

newsletter.yaml

name: Create weekly useful links page
on:
  workflow_dispatch:
  schedule:
    - cron: '0 12 * * 1' # At 12:00 on Monday

jobs:
  create_weekly_links:
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: 'true'
      - name: Tailscale
        uses: tailscale/github-action@v3
        with:
            oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
            oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
            tags: tag:link-collector
            use-cache: true
      
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'
      
      - name: Install Python dependencies
        run: |
          cd src/linkwarden
          pip install -r requirements.txt
      
      - name: Generate weekly links
        run: |
          cd src/linkwarden
          python add_links.py --host "${{ secrets.LINKWARDEN_HOST }}" --access-token "${{ secrets.LINKWARDEN_ACCESSTOKEN }}"
      
      - name: Get current date
        id: date
        run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
      
      - name: Commit and push changes
        uses: stefanzweifel/git-auto-commit-action@v5
        with:
          commit_message: 'Add weekly links for ${{ steps.date.outputs.date }}'
          file_pattern: 'content/links/**/*.md content/link-overview.md'