autoTweetFetch

gistfile1.txt

// ==UserScript==
// @name         Twitter Timeline Downloader (修复展开按钮 v11 - 简约输出版)
// @namespace    http://tampermonkey.net/
// @version      11.0
// @description  修复日语展开按钮识别 + 精确选择器 + 简约机器友好输出格式
// @author       You
// @match        https://twitter.com/home*
// @match        https://x.com/home*
// @grant        GM_setValue
// @grant        GM_getValue
// ==/UserScript==

(function() {
    'use strict';

    // ================= 配置区域 =================
    const CONFIG = {
        scrollDelay: 2000,
        scrollStep: 800,
        minTweetLength: 10,
        maxAttempts: 200,
        expandDelay: 2000,
        expandRetries: 2,
        showLog: true
    };

    const STORAGE_KEY = 'twitter_last_tweet_id';
    // ===========================================

    let downloadBtn = null;
    let stopBtn = null;
    let clearBtn = null;
    let infoLabel = null;
    let shouldStop = false;
    let isRunning = false;
    let capturedTweets = [];

    // 1. 创建控制按钮组
    function createButtons() {
        const container = document.createElement('div');
        container.style.position = 'fixed';
        container.style.bottom = '20px';
        container.style.right = '20px';
        container.style.zIndex = '9999';
        container.style.display = 'flex';
        container.style.gap = '10px';
        container.style.flexDirection = 'column';
        container.style.alignItems = 'flex-end';
        document.body.appendChild(container);

        infoLabel = document.createElement('div');
        infoLabel.style.background = 'rgba(0,0,0,0.8)';
        infoLabel.style.color = '#fff';
        infoLabel.style.padding = '6px 12px';
        infoLabel.style.borderRadius = '8px';
        infoLabel.style.fontSize = '11px';
        infoLabel.style.fontFamily = 'monospace';
        infoLabel.style.marginBottom = '5px';
        updateInfoLabel();
        container.appendChild(infoLabel);

        const btnRow = document.createElement('div');
        btnRow.style.display = 'flex';
        btnRow.style.gap = '10px';
        container.appendChild(btnRow);

        downloadBtn = createButton({
            text: '📥 开始抓取',
            bg: '#1d9bf0',
            hoverBg: '#1a8cd8',
            onclick: startExport
        });
        btnRow.appendChild(downloadBtn);

        stopBtn = createButton({
            text: '⏹️ 停止并下载',
            bg: '#e0245e',
            hoverBg: '#c41c50',
            onclick: stopAndDownload,
            display: 'none'
        });
        btnRow.appendChild(stopBtn);

        clearBtn = createButton({
            text: '🗑️ 清除断点',
            bg: '#536471',
            hoverBg: '#3f4b58',
            onclick: clearCheckpoint
        });
        btnRow.appendChild(clearBtn);
    }

    function createButton({ text, bg, hoverBg, onclick, display = 'block' }) {
        const btn = document.createElement('button');
        btn.innerText = text;
        btn.style.padding = '10px 16px';
        btn.style.background = bg;
        btn.style.color = 'white';
        btn.style.border = 'none';
        btn.style.borderRadius = '20px';
        btn.style.fontSize = '13px';
        btn.style.fontWeight = 'bold';
        btn.style.cursor = 'pointer';
        btn.style.boxShadow = '0 2px 8px rgba(0,0,0,0.3)';
        btn.style.display = display;
        btn.onmouseover = () => { if (!isRunning || btn === clearBtn) btn.style.background = hoverBg; };
        btn.onmouseout = () => { if (!isRunning || btn === clearBtn) btn.style.background = bg; };
        btn.onclick = onclick;
        return btn;
    }

    function updateInfoLabel() {
        const lastId = getLastTweetId();
        if (lastId) {
            infoLabel.innerText = `📍 断点：${lastId.substring(0, 12)}...`;
            infoLabel.style.color = '#1d9bf0';
        } else {
            infoLabel.innerText = '⚪ 无断点记录';
            infoLabel.style.color = '#888';
        }
    }

    // 2. 存储管理
    function getLastTweetId() {
        return GM_getValue(STORAGE_KEY, '');
    }

    function saveLastTweetId(tweetId) {
        GM_setValue(STORAGE_KEY, tweetId);
        log(`💾 已保存断点：${tweetId}`);
        updateInfoLabel();
    }

    // 3. 清除断点
    function clearCheckpoint() {
        if (isRunning) {
            alert('⚠️ 抓取进行中，请先停止');
            return;
        }

        const lastId = getLastTweetId();
        if (!lastId) {
            updateInfoLabel();
            return;
        }

        if (confirm(`🗑️ 确定要清除断点吗？\n\n上次抓取 ID: ${lastId}\n\n清除后下次将从头开始抓取。`)) {
            GM_setValue(STORAGE_KEY, '');
            updateInfoLabel();
            log('✅ 断点已清除');
        }
    }

    // 4. 开始抓取
    async function startExport() {
        if (isRunning) return;

        isRunning = true;
        shouldStop = false;
        capturedTweets = [];

        downloadBtn.style.display = 'none';
        stopBtn.style.display = 'block';
        clearBtn.style.display = 'none';

        try {
            window.scrollTo(0, 0);
            await new Promise(r => setTimeout(r, 1000));

            const lastId = getLastTweetId();
            log(`🎯 开始抓取，上次断点：${lastId || '无'}`);
            log(`ℹ️ 说明：转发的旧推文也会被抓取（按时间线出现顺序，非发布时间）`);

            await scrollUntilCheckpoint(lastId);

            if (capturedTweets.length === 0) {
                alert('⚠️ 没抓到新推文');
                resetButtons();
                return;
            }

            downloadToFile(capturedTweets, lastId);

            if (capturedTweets.length > 0) {
                saveLastTweetId(capturedTweets[capturedTweets.length - 1].tweetId);
            }

            showSuccessStatus(capturedTweets.length);
            resetButtons();

        } catch (error) {
            console.error('导出失败:', error);
            alert('❌ 导出失败，请查看控制台错误');
            resetButtons();
        }
    }

    function showSuccessStatus(count) {
        downloadBtn.innerText = `✅ 已导出 ${count} 条`;
        downloadBtn.style.background = '#00ba7c';
        setTimeout(() => {
            downloadBtn.innerText = '📥 开始抓取';
            downloadBtn.style.background = '#1d9bf0';
        }, 3000);
    }

    // 5. 手动停止并下载
    function stopAndDownload() {
        shouldStop = true;
        log('🛑 用户请求停止...');
    }

    // 6. 重置按钮
    function resetButtons() {
        isRunning = false;
        downloadBtn.style.display = 'block';
        stopBtn.style.display = 'none';
        clearBtn.style.display = 'block';
        stopBtn.innerText = '⏹️ 停止并下载';
    }

    // 7. 滚动直到遇到断点或手动停止
    async function scrollUntilCheckpoint(lastId) {
        let consecutiveNoNew = 0;
        let attempts = 0;
        let foundCheckpoint = false;

        const globalTweetIds = new Set();

        while (!shouldStop && attempts < CONFIG.maxAttempts) {
            attempts++;
            stopBtn.innerText = `⏹️ 停止 (${capturedTweets.length})`;

            window.scrollBy(0, CONFIG.scrollStep);
            await new Promise(r => setTimeout(r, CONFIG.scrollDelay));

            const expandedCount = await expandAllLongTweets();
            if (expandedCount > 0) {
                log(`🔓 展开了 ${expandedCount} 条长推文`);
            }

            const currentTweets = captureTweets();
            let newCount = 0;
            let duplicateInBatch = 0;
            let oldTweetCount = 0;

            for (const tweet of currentTweets) {
                if (lastId && tweet.tweetId === lastId) {
                    log(`✅ 遇到断点 ID: ${tweet.tweetId}`);
                    foundCheckpoint = true;
                    break;
                }

                if (globalTweetIds.has(tweet.tweetId)) {
                    duplicateInBatch++;
                    continue;
                }

                if (lastId && tweet.tweetId < lastId) {
                    oldTweetCount++;
                }

                globalTweetIds.add(tweet.tweetId);
                capturedTweets.push(tweet);
                newCount++;
            }

            if (foundCheckpoint) break;

            if (newCount > 0) {
                consecutiveNoNew = 0;
                log(`📜 第${attempts}次 | 新增${newCount}条 (含${oldTweetCount}条旧推文转发) | 重复${duplicateInBatch}条 | 总计${capturedTweets.length}条`);
            } else {
                consecutiveNoNew++;
                log(`⚠️ 第${attempts}次 | 无新增 | 连续${consecutiveNoNew}次`);
            }

            if (consecutiveNoNew >= 15) {
                log('⚠️ 连续 15 次无新内容，停止');
                break;
            }
        }

        if (shouldStop) {
            log('🛑 用户手动停止');
        }

        log(`✅ 抓取完成，共 ${capturedTweets.length} 条新推文`);
        return capturedTweets;
    }

    // 8. 修复：展开所有长推文（v11 精确选择器版）
    async function expandAllLongTweets() {
        let expandedCount = 0;
        const initialUrl = window.location.href;

        const allExpandButtons = document.querySelectorAll('[data-testid="tweet-text-show-more-link"]');

        log(`🔍 找到 ${allExpandButtons.length} 个展开按钮`);

        if (allExpandButtons.length === 0) {
            log(`ℹ️ 当前页面没有需要展开的长推文`);
            return 0;
        }

        const validButtons = [];

        for (const btn of allExpandButtons) {
            try {
                const article = btn.closest('article[data-testid="tweet"]');
                if (!article) continue;

                let hasNestedArticle = false;
                let current = btn.parentElement;
                while (current && current !== article) {
                    if (current.tagName === 'ARTICLE') {
                        hasNestedArticle = true;
                        break;
                    }
                    current = current.parentElement;
                }

                if (hasNestedArticle) continue;

                const quotedTweet = btn.closest('[data-testid="quoted-tweet"]');
                if (quotedTweet) continue;

                const conversationThread = btn.closest('[data-testid="conversationThread"]');
                if (conversationThread && conversationThread !== article) continue;

                const textElement = article.querySelector('[data-testid="tweetText"]');
                if (!textElement) continue;

                const btnParent = btn.parentElement;
                const textParent = textElement.parentElement;

                let sameContainer = false;
                let checkParent = btnParent;
                let depth = 0;
                while (checkParent && checkParent !== article && depth < 5) {
                    if (checkParent === textParent || checkParent.contains(textElement)) {
                        sameContainer = true;
                        break;
                    }
                    checkParent = checkParent.parentElement;
                    depth++;
                }

                if (!sameContainer) continue;

                validButtons.push({ btn, article, textElement });

            } catch (e) {
                log(`⚠️ 验证失败：${e.message}`);
            }
        }

        log(`📋 过滤后剩余 ${validButtons.length}/${allExpandButtons.length} 个有效按钮`);

        if (validButtons.length === 0) {
            log(`ℹ️ 没有需要展开的主推文`);
            return 0;
        }

        for (let i = 0; i < validButtons.length; i++) {
            const { btn, article, textElement } = validButtons[i];

            try {
                const text = btn.innerText || btn.textContent || '';

                if (text.includes('Show less') ||
                    text.includes('显示较少') ||
                    text.includes('表示を減らす')) {
                    continue;
                }

                const beforeLength = textElement.innerText.length;
                const beforeUrl = window.location.href;

                btn.scrollIntoView({ block: 'center', behavior: 'smooth' });
                await new Promise(r => setTimeout(r, 800));

                btn.click();
                await new Promise(r => setTimeout(r, 3000));

                if (window.location.href !== beforeUrl) {
                    window.history.back();
                    await new Promise(r => setTimeout(r, 2000));
                    continue;
                }

                const afterLength = textElement.innerText.length;
                const newText = btn.innerText || btn.textContent || '';

                if (afterLength > beforeLength) {
                    expandedCount++;
                } else if (newText.includes('Show less') || newText.includes('显示较少') || newText.includes('表示を減らす')) {
                    expandedCount++;
                }

                await new Promise(r => setTimeout(r, 500));

            } catch (e) {
                log(`❌ 按钮${i+1}: 错误 ${e.message}`);
            }
        }

        log(`🔓 共展开了 ${expandedCount}/${validButtons.length} 条长推文`);
        await new Promise(r => setTimeout(r, 500));
        return expandedCount;
    }

    // 9. 抓取推文
    function captureTweets() {
        const articles = document.querySelectorAll('article[data-testid="tweet"], article[role="article"]');
        const tweets = [];
        const seenIds = new Set();

        articles.forEach(article => {
            try {
                const textElement = article.querySelector('[data-testid="tweetText"]');
                const content = textElement ? textElement.innerText.trim() : '';

                const userContainer = article.querySelector('[data-testid="User-Name"]');
                let authorName = 'Unknown';
                let authorHandle = 'Unknown';

                if (userContainer) {
                    const spans = userContainer.querySelectorAll('span span');
                    if (spans.length >= 1) authorName = spans[0].innerText.trim();
                    if (spans.length >= 2) authorHandle = spans[1].innerText.trim().replace('@', '');
                }

                const timeElement = article.querySelector('time');
                const time = timeElement ? timeElement.dateTime : '';

                const linkElement = article.querySelector('a[href*="/status/"]');
                let tweetId = '';
                let tweetUrl = '';

                if (linkElement && linkElement.href) {
                    tweetUrl = linkElement.href;
                    const match = linkElement.href.match(/\/status\/(\d+)/);
                    if (match) tweetId = match[1];
                }

                if (!tweetId) {
                    tweetId = `${authorHandle}_${time}_${content.substring(0, 30)}`;
                }

                if (seenIds.has(tweetId)) return;
                seenIds.add(tweetId);

                if (content.length >= CONFIG.minTweetLength) {
                    tweets.push({
                        tweetId,
                        authorName,
                        authorHandle,
                        time,
                        content,
                        tweetUrl
                    });
                }
            } catch (e) {
                // 跳过
            }
        });

        return tweets;
    }

    // 10. 下载文件 ✅ 简约机器友好格式
    function downloadToFile(tweets, lastId) {
        const now = new Date();
        const timestamp = now.toISOString().replace(/[:.]/g, '-').slice(0, -5);
        const filename = `twitter_new_${tweets.length}条_${timestamp}.txt`;

        // ✅ 简约格式：字段换行分隔 + 横线分割每组推文
        const content = tweets.map((t, i) => {
            return `推文 #${String(i + 1).padStart(3, '0')}
作者：${t.authorName} (@${t.authorHandle})
时间：${t.time}
ID: ${t.tweetId}
链接：${t.tweetUrl || 'N/A'}
内容：
${t.content}
----------------------------------------`;
        }).join('\n');

        // ✅ 简化header，保留关键元数据（机器可解析）
        const header = `EXPORT_META
export_time: ${now.toISOString()}
tweet_count: ${tweets.length}
last_checkpoint_id: ${lastId || 'none'}
first_tweet_id: ${tweets[0]?.tweetId || 'none'}
last_tweet_id: ${tweets[tweets.length - 1]?.tweetId || 'none'}
---
`;

        const blob = new Blob([header + content], { type: 'text/plain;charset=utf-8' });
        const url = URL.createObjectURL(blob);
        const a = document.createElement('a');
        a.href = url;
        a.download = filename;
        document.body.appendChild(a);
        a.click();
        document.body.removeChild(a);
        URL.revokeObjectURL(url);
    }

    function formatTime(timeStr) {
        if (!timeStr) return 'N/A';
        try {
            const date = new Date(timeStr);
            return date.toLocaleString('zh-CN');
        } catch {
            return timeStr;
        }
    }

    function log(...args) {
        if (CONFIG.showLog) {
            console.log('[Twitter Downloader]', ...args);
        }
    }

    // 初始化
    createButtons();
    log('🚀 Twitter Downloader v11.0 已加载');
})();