316 lines
9.5 KiB
JavaScript
316 lines
9.5 KiB
JavaScript
/* eslint-disable no-console */
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
|
||
const { loadConfig } = require('../src/generator.js');
|
||
const { createLogger, isVerbose, startTimer } = require('../src/generator/utils/logger');
|
||
|
||
const log = createLogger('sync:projects');
|
||
|
||
const DEFAULT_SETTINGS = {
|
||
enabled: true,
|
||
cacheDir: 'dev',
|
||
fetch: {
|
||
timeoutMs: 10_000,
|
||
concurrency: 4,
|
||
userAgent: 'MeNavProjectsSync/1.0',
|
||
},
|
||
colors: {
|
||
url: 'https://raw.githubusercontent.com/ozh/github-colors/master/colors.json',
|
||
maxAgeMs: 7 * 24 * 60 * 60 * 1000,
|
||
},
|
||
};
|
||
|
||
function parseBooleanEnv(value, fallback) {
|
||
if (value === undefined || value === null || value === '') return fallback;
|
||
const v = String(value).trim().toLowerCase();
|
||
if (v === '1' || v === 'true' || v === 'yes' || v === 'y') return true;
|
||
if (v === '0' || v === 'false' || v === 'no' || v === 'n') return false;
|
||
return fallback;
|
||
}
|
||
|
||
function parseIntegerEnv(value, fallback) {
|
||
if (value === undefined || value === null || value === '') return fallback;
|
||
const n = Number.parseInt(String(value), 10);
|
||
return Number.isFinite(n) ? n : fallback;
|
||
}
|
||
|
||
function getSettings(config) {
|
||
const fromConfig =
|
||
config && config.site && config.site.github && typeof config.site.github === 'object'
|
||
? config.site.github
|
||
: {};
|
||
|
||
const merged = {
|
||
...DEFAULT_SETTINGS,
|
||
...fromConfig,
|
||
fetch: {
|
||
...DEFAULT_SETTINGS.fetch,
|
||
...(fromConfig.fetch || {}),
|
||
},
|
||
colors: {
|
||
...DEFAULT_SETTINGS.colors,
|
||
...(fromConfig.colors || {}),
|
||
},
|
||
};
|
||
|
||
merged.enabled = parseBooleanEnv(process.env.PROJECTS_ENABLED, merged.enabled);
|
||
merged.cacheDir = process.env.PROJECTS_CACHE_DIR
|
||
? String(process.env.PROJECTS_CACHE_DIR)
|
||
: merged.cacheDir;
|
||
merged.fetch.timeoutMs = parseIntegerEnv(
|
||
process.env.PROJECTS_FETCH_TIMEOUT,
|
||
merged.fetch.timeoutMs
|
||
);
|
||
merged.fetch.concurrency = parseIntegerEnv(
|
||
process.env.PROJECTS_FETCH_CONCURRENCY,
|
||
merged.fetch.concurrency
|
||
);
|
||
|
||
merged.fetch.timeoutMs = Math.max(1_000, merged.fetch.timeoutMs);
|
||
merged.fetch.concurrency = Math.max(1, Math.min(10, merged.fetch.concurrency));
|
||
|
||
return merged;
|
||
}
|
||
|
||
function ensureDir(dirPath) {
|
||
fs.mkdirSync(dirPath, { recursive: true });
|
||
}
|
||
|
||
function isGithubRepoUrl(url) {
|
||
if (!url) return null;
|
||
try {
|
||
const u = new URL(String(url));
|
||
if (u.protocol !== 'https:' && u.protocol !== 'http:') return null;
|
||
if (u.hostname.toLowerCase() !== 'github.com') return null;
|
||
const parts = u.pathname.split('/').filter(Boolean);
|
||
if (parts.length < 2) return null;
|
||
const owner = parts[0];
|
||
const repo = parts[1].replace(/\.git$/i, '');
|
||
if (!owner || !repo) return null;
|
||
return { owner, repo, canonicalUrl: `https://github.com/${owner}/${repo}` };
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function collectSitesRecursively(node, output) {
|
||
if (!node || typeof node !== 'object') return;
|
||
if (Array.isArray(node.subcategories))
|
||
node.subcategories.forEach((child) => collectSitesRecursively(child, output));
|
||
if (Array.isArray(node.groups))
|
||
node.groups.forEach((child) => collectSitesRecursively(child, output));
|
||
if (Array.isArray(node.subgroups))
|
||
node.subgroups.forEach((child) => collectSitesRecursively(child, output));
|
||
if (Array.isArray(node.sites)) node.sites.forEach((site) => output.push(site));
|
||
}
|
||
|
||
function findProjectsPages(config) {
|
||
const pages = [];
|
||
const nav = Array.isArray(config.navigation) ? config.navigation : [];
|
||
nav.forEach((item) => {
|
||
const pageId = item && item.id ? String(item.id) : '';
|
||
if (!pageId || !config[pageId]) return;
|
||
const page = config[pageId];
|
||
const templateName = page && page.template ? String(page.template) : pageId;
|
||
if (templateName !== 'projects') return;
|
||
pages.push({ pageId, page });
|
||
});
|
||
return pages;
|
||
}
|
||
|
||
async function fetchJsonWithTimeout(url, { timeoutMs, headers }) {
|
||
const controller = new AbortController();
|
||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||
try {
|
||
const response = await fetch(url, { method: 'GET', headers, signal: controller.signal });
|
||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||
return await response.json();
|
||
} finally {
|
||
clearTimeout(timer);
|
||
}
|
||
}
|
||
|
||
async function loadLanguageColors(settings, cacheBaseDir) {
|
||
const cachePath = path.join(cacheBaseDir, 'github-colors.json');
|
||
|
||
try {
|
||
const stat = fs.existsSync(cachePath) ? fs.statSync(cachePath) : null;
|
||
if (stat && stat.mtimeMs && Date.now() - stat.mtimeMs < settings.colors.maxAgeMs) {
|
||
const raw = fs.readFileSync(cachePath, 'utf8');
|
||
const parsed = JSON.parse(raw);
|
||
if (parsed && typeof parsed === 'object') return parsed;
|
||
}
|
||
} catch {
|
||
// 继续联网抓取
|
||
}
|
||
|
||
try {
|
||
const headers = { 'user-agent': settings.fetch.userAgent, accept: 'application/json' };
|
||
const colors = await fetchJsonWithTimeout(settings.colors.url, {
|
||
timeoutMs: settings.fetch.timeoutMs,
|
||
headers,
|
||
});
|
||
if (colors && typeof colors === 'object') {
|
||
fs.writeFileSync(cachePath, JSON.stringify(colors, null, 2), 'utf8');
|
||
return colors;
|
||
}
|
||
} catch (error) {
|
||
log.warn('获取语言颜色表失败（将不输出 languageColor）', {
|
||
message: String(error && error.message ? error.message : error),
|
||
});
|
||
}
|
||
|
||
return {};
|
||
}
|
||
|
||
async function fetchRepoMeta(repo, settings, colors) {
|
||
const headers = {
|
||
'user-agent': settings.fetch.userAgent,
|
||
accept: 'application/vnd.github+json',
|
||
};
|
||
|
||
const apiUrl = `https://api.github.com/repos/${repo.owner}/${repo.repo}`;
|
||
const data = await fetchJsonWithTimeout(apiUrl, { timeoutMs: settings.fetch.timeoutMs, headers });
|
||
|
||
const language = data && data.language ? String(data.language) : '';
|
||
const stars = data && Number.isFinite(data.stargazers_count) ? data.stargazers_count : null;
|
||
const forks = data && Number.isFinite(data.forks_count) ? data.forks_count : null;
|
||
|
||
let languageColor = '';
|
||
if (language && colors && colors[language] && colors[language].color) {
|
||
languageColor = String(colors[language].color);
|
||
}
|
||
|
||
return {
|
||
url: repo.canonicalUrl,
|
||
fullName: data && data.full_name ? String(data.full_name) : `${repo.owner}/${repo.repo}`,
|
||
language,
|
||
languageColor,
|
||
stars,
|
||
forks,
|
||
};
|
||
}
|
||
|
||
async function runPool(items, concurrency, worker) {
|
||
const results = [];
|
||
let index = 0;
|
||
|
||
async function runOne() {
|
||
while (index < items.length) {
|
||
const current = items[index];
|
||
index += 1;
|
||
// eslint-disable-next-line no-await-in-loop
|
||
const result = await worker(current);
|
||
if (result) results.push(result);
|
||
}
|
||
}
|
||
|
||
const runners = Array.from({ length: Math.min(concurrency, items.length) }, () => runOne());
|
||
await Promise.all(runners);
|
||
return results;
|
||
}
|
||
|
||
async function main() {
|
||
const elapsedMs = startTimer();
|
||
const config = loadConfig();
|
||
const settings = getSettings(config);
|
||
|
||
log.info('开始');
|
||
|
||
if (!settings.enabled) {
|
||
log.ok('projects 仓库同步已禁用，跳过', { env: 'PROJECTS_ENABLED=false' });
|
||
return;
|
||
}
|
||
|
||
const cacheBaseDir = path.isAbsolute(settings.cacheDir)
|
||
? settings.cacheDir
|
||
: path.join(process.cwd(), settings.cacheDir);
|
||
ensureDir(cacheBaseDir);
|
||
|
||
const colors = await loadLanguageColors(settings, cacheBaseDir);
|
||
const pages = findProjectsPages(config);
|
||
|
||
if (!pages.length) {
|
||
log.ok('未找到 template=projects 的页面，跳过同步');
|
||
return;
|
||
}
|
||
|
||
log.info('准备同步 projects 页面缓存', { pages: pages.length });
|
||
|
||
let pageSuccess = 0;
|
||
let pageFailed = 0;
|
||
|
||
for (const { pageId, page } of pages) {
|
||
const categories = Array.isArray(page.categories) ? page.categories : [];
|
||
const sites = [];
|
||
categories.forEach((category) => collectSitesRecursively(category, sites));
|
||
|
||
const repos = sites
|
||
.map((site) => (site && site.url ? isGithubRepoUrl(site.url) : null))
|
||
.filter(Boolean);
|
||
|
||
const unique = new Map();
|
||
repos.forEach((r) => unique.set(r.canonicalUrl, r));
|
||
const repoList = Array.from(unique.values());
|
||
|
||
if (!repoList.length) {
|
||
log.ok('页面未发现 GitHub 仓库链接，跳过', { page: pageId });
|
||
continue;
|
||
}
|
||
|
||
let success = 0;
|
||
let failed = 0;
|
||
|
||
const results = await runPool(repoList, settings.fetch.concurrency, async (repo) => {
|
||
try {
|
||
const meta = await fetchRepoMeta(repo, settings, colors);
|
||
success += 1;
|
||
return meta;
|
||
} catch (error) {
|
||
failed += 1;
|
||
log.warn('拉取仓库元信息失败（best-effort）', {
|
||
repo: repo.canonicalUrl,
|
||
message: String(error && error.message ? error.message : error),
|
||
});
|
||
return null;
|
||
}
|
||
});
|
||
|
||
const payload = {
|
||
version: '1.0',
|
||
pageId,
|
||
generatedAt: new Date().toISOString(),
|
||
repos: results,
|
||
stats: {
|
||
totalRepos: repoList.length,
|
||
success,
|
||
failed,
|
||
},
|
||
};
|
||
|
||
const cachePath = path.join(cacheBaseDir, `${pageId}.repo-cache.json`);
|
||
fs.writeFileSync(cachePath, JSON.stringify(payload, null, 2), 'utf8');
|
||
|
||
if (failed === 0) pageSuccess += 1;
|
||
else pageFailed += 1;
|
||
|
||
log.ok('页面同步完成', {
|
||
page: pageId,
|
||
success,
|
||
failed,
|
||
cache: cachePath,
|
||
});
|
||
}
|
||
|
||
log.ok('完成', { ms: elapsedMs(), pages: pages.length, pageSuccess, pageFailed });
|
||
}
|
||
|
||
main().catch((error) => {
|
||
log.error('执行异常（best-effort，不阻断后续 build）', {
|
||
message: error && error.message ? error.message : String(error),
|
||
});
|
||
if (isVerbose() && error && error.stack) console.error(error.stack);
|
||
process.exitCode = 0; // best-effort：不阻断后续 build
|
||
});
|