feat: add Feed background RSS fetch job and /feed/articles endpoint
This commit is contained in:
parent
ac8cc6db81
commit
c7b6d0b2d3
178
server.js
178
server.js
|
|
@ -22,6 +22,8 @@ const crypto = require('crypto');
|
|||
const jwt = require('jsonwebtoken');
|
||||
const os = require('os');
|
||||
const { execSync } = require('child_process');
|
||||
let RssParser = null;
|
||||
try { RssParser = require('rss-parser'); } catch (_) { console.warn('[Feed] rss-parser not found, background fetch disabled'); }
|
||||
|
||||
// ── Auth: WebAuthn (ESM dynamic import) ─────────────────────────────
|
||||
let webauthn = null;
|
||||
|
|
@ -1888,8 +1890,13 @@ ${excerpt}
|
|||
|
||||
r.get('/feed/media', authMiddleware, async (req, res) => {
|
||||
try {
|
||||
// default_user のメディア（共通）＋ ユーザー自身のメディアを統合して返す
|
||||
const result = await pool.query(
|
||||
'SELECT id, name, feed_url, site_url, category, is_active, created_at FROM feed_media WHERE user_id=$1 ORDER BY created_at ASC',
|
||||
`SELECT id, name, feed_url, site_url, category, is_active, created_at,
|
||||
(user_id = 'default_user') AS is_default
|
||||
FROM feed_media
|
||||
WHERE user_id = 'default_user' OR user_id = $1
|
||||
ORDER BY is_default DESC, created_at ASC`,
|
||||
[req.userId]
|
||||
);
|
||||
res.json(result.rows);
|
||||
|
|
@ -1897,15 +1904,27 @@ ${excerpt}
|
|||
});
|
||||
|
||||
r.post('/feed/media', authMiddleware, async (req, res) => {
|
||||
const { name, feed_url, site_url = '', category = 'tech', is_active = true } = req.body || {};
|
||||
const { name, feed_url, site_url = '', is_active = true } = req.body || {};
|
||||
if (!name || !feed_url) return res.status(400).json({ error: 'name and feed_url required' });
|
||||
// カテゴリ自動判定（指定があればそのまま使用）
|
||||
let category = req.body.category || '';
|
||||
if (!category) {
|
||||
const urlLower = feed_url.toLowerCase();
|
||||
if (/news|nhk|yahoo|nikkei|asahi|mainichi|yomiuri/.test(urlLower)) category = 'news';
|
||||
else if (/business|bizjapan|diamond|toyo|kaizen/.test(urlLower)) category = 'business';
|
||||
else if (/lifestyle|life|food|cooking|fashion|beauty|travel/.test(urlLower)) category = 'lifestyle';
|
||||
else category = 'tech';
|
||||
}
|
||||
try {
|
||||
const result = await pool.query(
|
||||
'INSERT INTO feed_media (user_id, name, feed_url, site_url, category, is_active) VALUES ($1,$2,$3,$4,$5,$6) RETURNING id, name, feed_url, site_url, category, is_active, created_at',
|
||||
[req.userId, name, feed_url, site_url, category, is_active]
|
||||
);
|
||||
res.status(201).json(result.rows[0]);
|
||||
} catch (e) { console.error(e); res.status(500).json({ error: 'DB error' }); }
|
||||
} catch (e) {
|
||||
if (e.code === '23505') return res.status(409).json({ error: 'このメディアはすでに追加済みです' });
|
||||
console.error(e); res.status(500).json({ error: 'DB error' });
|
||||
}
|
||||
});
|
||||
|
||||
r.patch('/feed/media/:id', authMiddleware, async (req, res) => {
|
||||
|
|
@ -1940,6 +1959,81 @@ ${excerpt}
|
|||
} catch (e) { console.error(e); res.status(500).json({ error: 'DB error' }); }
|
||||
});
|
||||
|
||||
// ── Feed Articles（DBキャッシュから高速配信）──────────────────────
|
||||
// VPS背景取得ジョブがfeed_articlesテーブルに書き込み、ここで読む
|
||||
r.get('/feed/articles', authMiddleware, async (req, res) => {
|
||||
try {
|
||||
const limit = Math.min(parseInt(req.query.limit || '100'), 200);
|
||||
const offset = parseInt(req.query.offset || '0');
|
||||
const category = req.query.category || null;
|
||||
const mediaId = req.query.media_id || null;
|
||||
|
||||
let where = `WHERE fa.user_id IN ('default_user', $1)`;
|
||||
const vals = [req.userId];
|
||||
let idx = 2;
|
||||
|
||||
if (category) { where += ` AND fm.category = $${idx++}`; vals.push(category); }
|
||||
if (mediaId) { where += ` AND fa.media_id = $${idx++}`; vals.push(mediaId); }
|
||||
|
||||
const [articlesResult, mediasResult] = await Promise.all([
|
||||
pool.query(
|
||||
`SELECT fa.id, fa.url, fa.title, fa.summary, fa.author,
|
||||
fa.published_at, fa.is_read,
|
||||
fm.id AS media_id, fm.name AS source, fm.category,
|
||||
fm.feed_url, fm.site_url, fm.favicon
|
||||
FROM feed_articles fa
|
||||
JOIN feed_media fm ON fa.media_id = fm.id
|
||||
${where}
|
||||
ORDER BY fa.published_at DESC NULLS LAST
|
||||
LIMIT $${idx} OFFSET $${idx + 1}`,
|
||||
[...vals, limit, offset]
|
||||
),
|
||||
pool.query(
|
||||
`SELECT id, name, feed_url, site_url, category, favicon,
|
||||
(user_id = 'default_user') AS is_default
|
||||
FROM feed_media
|
||||
WHERE user_id IN ('default_user', $1) AND is_active = true
|
||||
ORDER BY is_default DESC, created_at ASC`,
|
||||
[req.userId]
|
||||
)
|
||||
]);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
articles: articlesResult.rows,
|
||||
medias: mediasResult.rows,
|
||||
categories: [
|
||||
{ id: 'all', name: '全て', icon: 'layout-grid' },
|
||||
{ id: 'news', name: 'ニュース', icon: 'newspaper' },
|
||||
{ id: 'tech', name: 'テクノロジー', icon: 'rocket' },
|
||||
{ id: 'lifestyle', name: 'ライフスタイル', icon: 'coffee' },
|
||||
{ id: 'business', name: 'ビジネス', icon: 'briefcase' },
|
||||
],
|
||||
total: articlesResult.rows.length,
|
||||
updatedAt: new Date().toISOString()
|
||||
});
|
||||
} catch (e) { console.error(e); res.status(500).json({ error: 'DB error' }); }
|
||||
});
|
||||
|
||||
// Feed既読マーク
|
||||
r.patch('/feed/articles/:id/read', authMiddleware, async (req, res) => {
|
||||
try {
|
||||
await pool.query(
|
||||
`UPDATE feed_articles SET is_read = true WHERE id = $1 AND user_id IN ('default_user', $2)`,
|
||||
[req.params.id, req.userId]
|
||||
);
|
||||
res.json({ ok: true });
|
||||
} catch (e) { console.error(e); res.status(500).json({ error: 'DB error' }); }
|
||||
});
|
||||
|
||||
// 手動リフレッシュ（フロントの更新ボタンから呼ぶ）
|
||||
r.post('/feed/refresh', authMiddleware, async (req, res) => {
|
||||
try {
|
||||
const count = await runFeedFetch();
|
||||
res.json({ ok: true, fetched: count });
|
||||
} catch (e) { console.error(e); res.status(500).json({ error: e.message }); }
|
||||
});
|
||||
|
||||
// ── TTS (VOICEVOX) ─────────────────────────────────────────────
|
||||
// VOICEVOX_URL: docker-compose.yml で設定（同ネットワーク内なら http://voicevox:50021）
|
||||
// 未設定時は localhost フォールバック（NAS上で同じコンテナホストの場合）
|
||||
|
|
@ -2802,6 +2896,76 @@ app.use('/api', router); // ローカル直接アクセス
|
|||
// ── 起動 ──────────────────────────────────
|
||||
const PORT = parseInt(process.env.PORT || '8090');
|
||||
|
||||
// ── Feed 背景取得ジョブ ────────────────────────────────────────────
|
||||
// feed_media テーブルの全URLを15分ごとに取得し feed_articles へ upsert
|
||||
let feedFetchRunning = false;
|
||||
|
||||
async function runFeedFetch() {
|
||||
if (!RssParser) return 0;
|
||||
if (feedFetchRunning) { console.log('[Feed] fetch already running, skip'); return 0; }
|
||||
feedFetchRunning = true;
|
||||
let totalNew = 0;
|
||||
try {
|
||||
const rssParser = new RssParser({
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (compatible; Posimai/1.0)',
|
||||
'Accept': 'application/rss+xml, application/atom+xml, application/xml, text/xml, */*'
|
||||
},
|
||||
timeout: 10000
|
||||
});
|
||||
|
||||
const mediasResult = await pool.query(
|
||||
`SELECT id, user_id, name, feed_url, category FROM feed_media WHERE is_active = true`
|
||||
);
|
||||
const medias = mediasResult.rows;
|
||||
console.log(`[Feed] fetching ${medias.length} feeds...`);
|
||||
|
||||
await Promise.allSettled(medias.map(async (media) => {
|
||||
try {
|
||||
const feed = await rssParser.parseURL(media.feed_url);
|
||||
const items = (feed.items || []).slice(0, 20);
|
||||
for (const item of items) {
|
||||
if (!item.link) continue;
|
||||
const publishedAt = item.pubDate || item.isoDate || null;
|
||||
try {
|
||||
const result = await pool.query(
|
||||
`INSERT INTO feed_articles (media_id, user_id, url, title, summary, author, published_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (user_id, url) DO NOTHING`,
|
||||
[
|
||||
media.id, media.user_id,
|
||||
item.link,
|
||||
(item.title || '').slice(0, 500),
|
||||
(item.contentSnippet || item.content || item.description || '').slice(0, 1000),
|
||||
(item.creator || item.author || '').slice(0, 200),
|
||||
publishedAt ? new Date(publishedAt) : null
|
||||
]
|
||||
);
|
||||
totalNew += result.rowCount;
|
||||
} catch (_) { /* duplicate or DB error, skip */ }
|
||||
}
|
||||
// last_fetched_at 更新
|
||||
await pool.query(
|
||||
`UPDATE feed_media SET last_fetched_at = NOW() WHERE id = $1`,
|
||||
[media.id]
|
||||
);
|
||||
} catch (e) {
|
||||
console.warn(`[Feed] failed to fetch ${media.feed_url}: ${e.message}`);
|
||||
}
|
||||
}));
|
||||
|
||||
// 古い記事を削除（30日以上前 + 既読）
|
||||
await pool.query(
|
||||
`DELETE FROM feed_articles WHERE published_at < NOW() - INTERVAL '30 days' AND is_read = true`
|
||||
);
|
||||
|
||||
console.log(`[Feed] fetch done. new articles: ${totalNew}`);
|
||||
} finally {
|
||||
feedFetchRunning = false;
|
||||
}
|
||||
return totalNew;
|
||||
}
|
||||
|
||||
loadWebauthn()
|
||||
.then(() => initDB())
|
||||
.then(() => {
|
||||
|
|
@ -2814,6 +2978,14 @@ loadWebauthn()
|
|||
console.log(` Local: http://localhost:${PORT}/api/health`);
|
||||
console.log(` Public: https://api.soar-enrich.com/brain/api/health`);
|
||||
});
|
||||
// 起動直後に1回取得し、以降15分ごとに繰り返す
|
||||
if (RssParser) {
|
||||
setTimeout(() => runFeedFetch().catch(e => console.error('[Feed] initial fetch error:', e.message)), 5000);
|
||||
setInterval(() => runFeedFetch().catch(e => console.error('[Feed] interval fetch error:', e.message)), 15 * 60 * 1000);
|
||||
console.log(' Feed: background fetch enabled (15min interval)');
|
||||
} else {
|
||||
console.log(' Feed: background fetch disabled (rss-parser not installed)');
|
||||
}
|
||||
})
|
||||
.catch(err => {
|
||||
console.error('[FATAL] Startup failed:', err.message);
|
||||
|
|
|
|||
Loading…
Reference in New Issue